Examples with DBAdapter org.smartdata.server.metastore.DBAdapter used on opensource projects

Example 6 with DBAdapter

use of org.smartdata.server.metastore.DBAdapter in project SSM by Intel-bigdata.

the class TestSmartRuleParser method parseAndExecuteRule.

private void parseAndExecuteRule(String rule) throws Exception {
    System.out.println("--> " + rule);
    InputStream input = new ByteArrayInputStream(rule.getBytes());
    ANTLRInputStream antlrInput = new ANTLRInputStream(input);
    SmartRuleLexer lexer = new SmartRuleLexer(antlrInput);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SmartRuleParser parser = new SmartRuleParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(new SSMRuleErrorListener());
    ParseTree tree = parser.ssmrule();
    System.out.println("Parser tree: " + tree.toStringTree(parser));
    System.out.println("Total number of errors: " + parseErrors.size());
    SmartRuleVisitTranslator visitor = new SmartRuleVisitTranslator();
    visitor.visit(tree);
    System.out.println("\nQuery:");
    TranslateResult result = visitor.generateSql();
    int index = 1;
    for (String sql : result.getSqlStatements()) {
        System.out.println("" + index + ". " + sql);
        index++;
    }
    if (parseErrors.size() > 0) {
        throw new IOException("Error while parse rule");
    }
    ExecutionContext ctx = new ExecutionContext();
    ctx.setProperty(ExecutionContext.RULE_ID, 2016);
    DBAdapter dbAdapter = new DBAdapter(TestDBUtil.getTestDBInstance());
    RuleQueryExecutor qe = new RuleQueryExecutor(null, ctx, result, dbAdapter);
    List<String> paths = qe.executeFileRuleQuery();
    index = 1;
    System.out.println("\nFiles:");
    for (String path : paths) {
        System.out.println("" + index + ". " + path);
        index++;
    }
}

Example 7 with DBAdapter

use of org.smartdata.server.metastore.DBAdapter in project SSM by Intel-bigdata.

the class TestInotifyEventApplier method testApplier.

@Test
public void testApplier() throws Exception {
    DFSClient client = mock(DFSClient.class);
    Connection connection = databaseTester.getConnection().getConnection();
    Util.initializeDataBase(connection);
    DBAdapter adapter = new DBAdapter(connection);
    InotifyEventApplier applier = new InotifyEventApplier(adapter, client);
    Event.CreateEvent createEvent = new Event.CreateEvent.Builder().iNodeType(Event.CreateEvent.INodeType.FILE).ctime(1).defaultBlockSize(1024).groupName("cg1").overwrite(true).ownerName("user1").path("/file").perms(new FsPermission("777")).replication(3).build();
    HdfsFileStatus status1 = new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission((short) 777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, (byte) 0);
    when(client.getFileInfo(anyString())).thenReturn(status1);
    applier.apply(new Event[] { createEvent });
    ResultSet result1 = adapter.executeQuery("SELECT * FROM files");
    Assert.assertEquals(result1.getString("path"), "/file");
    Assert.assertEquals(result1.getLong("fid"), 1010L);
    Assert.assertEquals(result1.getShort("permission"), 511);
    Event close = new Event.CloseEvent("/file", 1024, 0);
    applier.apply(new Event[] { close });
    ResultSet result2 = adapter.executeQuery("SELECT * FROM files");
    Assert.assertEquals(result2.getLong("length"), 1024);
    Assert.assertEquals(result2.getLong("modification_time"), 0L);
    //    Event truncate = new Event.TruncateEvent("/file", 512, 16);
    //    applier.apply(new Event[] {truncate});
    //    ResultSet result3 = adapter.executeQuery("SELECT * FROM files");
    //    Assert.assertEquals(result3.getLong("length"), 512);
    //    Assert.assertEquals(result3.getLong("modification_time"), 16L);
    Event meta = new Event.MetadataUpdateEvent.Builder().path("/file").metadataType(Event.MetadataUpdateEvent.MetadataType.TIMES).mtime(2).atime(3).replication(4).ownerName("user2").groupName("cg2").build();
    applier.apply(new Event[] { meta });
    ResultSet result4 = adapter.executeQuery("SELECT * FROM files");
    Assert.assertEquals(result4.getLong("access_time"), 3);
    Assert.assertEquals(result4.getLong("modification_time"), 2);
    Event.CreateEvent createEvent2 = new Event.CreateEvent.Builder().iNodeType(Event.CreateEvent.INodeType.DIRECTORY).ctime(1).groupName("cg1").overwrite(true).ownerName("user1").path("/dir").perms(new FsPermission("777")).replication(3).build();
    Event.CreateEvent createEvent3 = new Event.CreateEvent.Builder().iNodeType(Event.CreateEvent.INodeType.FILE).ctime(1).groupName("cg1").overwrite(true).ownerName("user1").path("/dir/file").perms(new FsPermission("777")).replication(3).build();
    Event rename = new Event.RenameEvent.Builder().dstPath("/dir2").srcPath("/dir").timestamp(5).build();
    applier.apply(new Event[] { createEvent2, createEvent3, rename });
    ResultSet result5 = adapter.executeQuery("SELECT * FROM files");
    List<String> expectedPaths = Arrays.asList("/dir2", "/dir2/file", "/file");
    List<String> actualPaths = new ArrayList<>();
    while (result5.next()) {
        actualPaths.add(result5.getString("path"));
    }
    Collections.sort(actualPaths);
    Assert.assertTrue(actualPaths.size() == 3);
    Assert.assertTrue(actualPaths.containsAll(expectedPaths));
    Event unlink = new Event.UnlinkEvent.Builder().path("/").timestamp(6).build();
    applier.apply(new Event[] { unlink });
    ResultSet result6 = adapter.executeQuery("SELECT * FROM files");
    Assert.assertFalse(result6.next());
}

Example 8 with DBAdapter

use of org.smartdata.server.metastore.DBAdapter in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testGetTables.

@Test
public void testGetTables() throws SQLException {
    DBAdapter adapter = mock(DBAdapter.class);
    TableEvictor tableEvictor = new CountEvictor(20);
    Map<TimeGranularity, AccessCountTableDeque> map = new HashMap<>();
    AccessCountTableDeque dayDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstDay = new AccessCountTable(0L, Constants.ONE_DAY_IN_MILLIS);
    dayDeque.add(firstDay);
    map.put(TimeGranularity.DAY, dayDeque);
    AccessCountTableDeque hourDeque = new AccessCountTableDeque(tableEvictor);
    AccessCountTable firstHour = new AccessCountTable(23 * Constants.ONE_HOUR_IN_MILLIS, 24 * Constants.ONE_HOUR_IN_MILLIS);
    AccessCountTable secondHour = new AccessCountTable(24 * Constants.ONE_HOUR_IN_MILLIS, 25 * Constants.ONE_HOUR_IN_MILLIS);
    hourDeque.add(firstHour);
    hourDeque.add(secondHour);
    map.put(TimeGranularity.HOUR, hourDeque);
    AccessCountTableDeque minuteDeque = new AccessCountTableDeque(tableEvictor);
    Integer numMins = 25 * 60;
    AccessCountTable firstMin = new AccessCountTable((numMins - 1) * Constants.ONE_MINUTE_IN_MILLIS, numMins * Constants.ONE_MINUTE_IN_MILLIS);
    AccessCountTable secondMin = new AccessCountTable(numMins * Constants.ONE_MINUTE_IN_MILLIS, (numMins + 1) * Constants.ONE_MINUTE_IN_MILLIS);
    minuteDeque.add(firstMin);
    minuteDeque.add(secondMin);
    map.put(TimeGranularity.MINUTE, minuteDeque);
    AccessCountTableDeque secondDeque = new AccessCountTableDeque(tableEvictor);
    Integer numSeconds = (25 * 60 + 1) * 60;
    AccessCountTable firstFiveSeconds = new AccessCountTable((numSeconds - 5) * Constants.ONE_SECOND_IN_MILLIS, numSeconds * Constants.ONE_SECOND_IN_MILLIS);
    AccessCountTable secondFiveSeconds = new AccessCountTable(numSeconds * Constants.ONE_SECOND_IN_MILLIS, (numSeconds + 5) * Constants.ONE_SECOND_IN_MILLIS);
    secondDeque.add(firstFiveSeconds);
    secondDeque.add(secondFiveSeconds);
    map.put(TimeGranularity.SECOND, secondDeque);
    List<AccessCountTable> firstResult = AccessCountTableManager.getTables(map, adapter, (numSeconds + 5) * Constants.ONE_SECOND_IN_MILLIS);
    Assert.assertTrue(firstResult.size() == 4);
    Assert.assertEquals(firstResult.get(0), firstDay);
    Assert.assertEquals(firstResult.get(1), secondHour);
    Assert.assertEquals(firstResult.get(2), secondMin);
    Assert.assertEquals(firstResult.get(3), secondFiveSeconds);
    List<AccessCountTable> secondResult = AccessCountTableManager.getTables(map, adapter, numSeconds * Constants.ONE_SECOND_IN_MILLIS);
    Assert.assertTrue(secondResult.size() == 4);
    AccessCountTable expectDay = new AccessCountTable(5 * Constants.ONE_SECOND_IN_MILLIS, Constants.ONE_DAY_IN_MILLIS);
    Assert.assertEquals(expectDay, secondResult.get(0));
    List<AccessCountTable> thirdResult = AccessCountTableManager.getTables(map, adapter, secondFiveSeconds.getEndTime() - 23 * Constants.ONE_HOUR_IN_MILLIS);
    Assert.assertTrue(thirdResult.size() == 4);
    Assert.assertEquals(thirdResult.get(0), firstHour);
    List<AccessCountTable> fourthResult = AccessCountTableManager.getTables(map, adapter, secondFiveSeconds.getEndTime() - 24 * Constants.ONE_HOUR_IN_MILLIS);
    Assert.assertTrue(fourthResult.size() == 3);
    Assert.assertEquals(fourthResult.get(0), secondHour);
}

Example 9 with DBAdapter

use of org.smartdata.server.metastore.DBAdapter in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testAddAccessCountInfo.

@Test
public void testAddAccessCountInfo() throws Exception {
    createTables(databaseTester.getConnection());
    IDataSet dataSet = new XmlDataSet(getClass().getClassLoader().getResourceAsStream("files.xml"));
    databaseTester.setDataSet(dataSet);
    databaseTester.onSetup();
    DBAdapter adapter = new DBAdapter(databaseTester.getConnection().getConnection());
    AccessCountTableManager manager = new AccessCountTableManager(adapter);
    List<FileAccessEvent> accessEvents = new ArrayList<>();
    accessEvents.add(new FileAccessEvent("file1", 0));
    accessEvents.add(new FileAccessEvent("file2", 1));
    accessEvents.add(new FileAccessEvent("file2", 2));
    accessEvents.add(new FileAccessEvent("file3", 2));
    accessEvents.add(new FileAccessEvent("file3", 3));
    accessEvents.add(new FileAccessEvent("file3", 4));
    accessEvents.add(new FileAccessEvent("file3", 5000));
    manager.onAccessEventsArrived(accessEvents);
    AccessCountTable accessCountTable = new AccessCountTable(0L, 5000L);
    ITable actual = databaseTester.getConnection().createTable(accessCountTable.getTableName());
    ITable expect = databaseTester.getDataSet().getTable("expect1");
    SortedTable sortedActual = new SortedTable(actual, new String[] { "fid" });
    sortedActual.setUseComparable(true);
    Assertion.assertEquals(expect, sortedActual);
}

Example 10 with DBAdapter

use of org.smartdata.server.metastore.DBAdapter in project SSM by Intel-bigdata.

the class TestAccessCountTableManager method testAccessCountTableManager.

@Test
public void testAccessCountTableManager() throws InterruptedException {
    DBAdapter adapter = mock(DBAdapter.class);
    AccessCountTableManager manager = new AccessCountTableManager(adapter);
    Long firstDayEnd = 24 * 60 * 60 * 1000L;
    AccessCountTable accessCountTable = new AccessCountTable(firstDayEnd - 5 * 1000, firstDayEnd, TimeGranularity.SECOND);
    manager.addTable(accessCountTable);
    Thread.sleep(5000);
    Map<TimeGranularity, AccessCountTableDeque> map = manager.getTableDeques();
    AccessCountTableDeque second = map.get(TimeGranularity.SECOND);
    Assert.assertTrue(second.size() == 1);
    Assert.assertEquals(second.peek(), accessCountTable);
    AccessCountTableDeque minute = map.get(TimeGranularity.MINUTE);
    AccessCountTable minuteTable = new AccessCountTable(firstDayEnd - 60 * 1000, firstDayEnd, TimeGranularity.MINUTE);
    Assert.assertTrue(minute.size() == 1);
    Assert.assertEquals(minute.peek(), minuteTable);
    AccessCountTableDeque hour = map.get(TimeGranularity.HOUR);
    AccessCountTable hourTable = new AccessCountTable(firstDayEnd - 60 * 60 * 1000, firstDayEnd, TimeGranularity.HOUR);
    Assert.assertTrue(hour.size() == 1);
    Assert.assertEquals(hour.peek(), hourTable);
    AccessCountTableDeque day = map.get(TimeGranularity.DAY);
    AccessCountTable dayTable = new AccessCountTable(firstDayEnd - 24 * 60 * 60 * 1000, firstDayEnd, TimeGranularity.DAY);
    Assert.assertTrue(day.size() == 1);
    Assert.assertEquals(day.peek(), dayTable);
}