Examples with RawStore org.apache.hadoop.hive.metastore.RawStore used on opensource projects

Example 6 with RawStore

use of org.apache.hadoop.hive.metastore.RawStore in project hive by apache.

the class TestHBaseImport method parallelOdd.

// Same as the test above except we create 9 of everything instead of 10.  This is important
// because in using a batch size of 2 the previous test guarantees 10 /2 =5 , meaning we'll
// have 5 writes on the partition queue with exactly 2 entries.  In this test we'll handle the
// case where the last entry in the queue has fewer partitions.
@Test
public void parallelOdd() throws Exception {
    int parallelFactor = 9;
    RawStore rdbms;
    rdbms = new ObjectStore();
    rdbms.setConf(conf);
    String[] dbNames = new String[] { "oddparalleldb1" };
    int now = (int) System.currentTimeMillis() / 1000;
    for (int i = 0; i < dbNames.length; i++) {
        rdbms.createDatabase(new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
        List<FieldSchema> cols = new ArrayList<>();
        cols.add(new FieldSchema("col1", "int", "nocomment"));
        SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
        StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
        List<FieldSchema> partCols = new ArrayList<>();
        partCols.add(new FieldSchema("region", "string", ""));
        for (int j = 0; j < parallelFactor; j++) {
            rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols, emptyParameters, null, null, null));
            for (int k = 0; k < parallelFactor; k++) {
                StorageDescriptor psd = new StorageDescriptor(sd);
                psd.setLocation("file:/tmp/region=" + k);
                Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j, now, now, psd, emptyParameters);
                rdbms.addPartition(part);
            }
        }
    }
    HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
    importer.setConnections(rdbms, store);
    importer.run();
    for (int i = 0; i < dbNames.length; i++) {
        Database db = store.getDatabase(dbNames[i]);
        Assert.assertNotNull(db);
        for (int j = 0; j < parallelFactor; j++) {
            Table table = store.getTable(db.getName(), "t" + j);
            Assert.assertNotNull(table);
            Assert.assertEquals(now, table.getLastAccessTime());
            Assert.assertEquals("input", table.getSd().getInputFormat());
            for (int k = 0; k < parallelFactor; k++) {
                Partition part = store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
                Assert.assertNotNull(part);
                Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
            }
            Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
        }
        Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
    }
}

Example 7 with RawStore

use of org.apache.hadoop.hive.metastore.RawStore in project metacat by Netflix.

the class MetacatHMSHandler method getMS.

@Override
public RawStore getMS() throws MetaException {
    RawStore ms = threadLocalMS.get();
    if (ms == null) {
        ms = newRawStore();
        ms.verifySchema();
        threadLocalMS.set(ms);
        ms = threadLocalMS.get();
    }
    return ms;
}

Example 8 with RawStore

use of org.apache.hadoop.hive.metastore.RawStore in project metacat by Netflix.

the class MetacatHMSHandler method setConf.

@Override
public void setConf(final Configuration conf) {
    threadLocalConf.set(conf);
    final RawStore ms = threadLocalMS.get();
    if (ms != null) {
        // reload if DS related configuration is changed
        ms.setConf(conf);
    }
}

Example 9 with RawStore

use of org.apache.hadoop.hive.metastore.RawStore in project hive by apache.

the class TestHBaseImport method importOneTableNonPartitioned.

@Test
public void importOneTableNonPartitioned() throws Exception {
    RawStore rdbms;
    rdbms = new ObjectStore();
    rdbms.setConf(conf);
    String[] dbNames = new String[] { "onetabdb1", "onetabdb2" };
    String[] roles = new String[] { "onetabrole1", "onetabrole2" };
    String[] tokenIds = new String[] { "onetabtokenid1", "onetabtokenid2" };
    String[] tokens = new String[] { "onetabtoken1", "onetabtoken2" };
    String[] masterKeys = new String[] { "onetabmk1", "onetabmk2" };
    int now = (int) System.currentTimeMillis() / 1000;
    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
    int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 : store.getAllTokenIdentifiers().size();
    int baseNumKeys = store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
    // Create the database so I can put the table in it.
    store.createDatabase(new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
    HBaseImport importer = new HBaseImport("-t", dbNames[0] + "." + tableNames[0]);
    importer.setConnections(rdbms, store);
    importer.run();
    // Make sure there aren't any extra roles
    Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
    Database db = store.getDatabase(dbNames[0]);
    Assert.assertNotNull(db);
    Table table = store.getTable(db.getName(), tableNames[0]);
    Assert.assertNotNull(table);
    Assert.assertEquals(1, store.getAllTables(db.getName()).size());
    Assert.assertNull(store.getTable(db.getName(), tableNames[1]));
    List<Index> indexes = store.getIndexes(db.getName(), tableNames[0], -1);
    Assert.assertEquals(2, indexes.size());
    Assert.assertEquals(0, store.getFunctions(dbNames[0], "*").size());
    Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
    Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
    String[] hbaseKeys = store.getMasterKeys();
    Assert.assertEquals(baseNumKeys, hbaseKeys.length);
}

Example 10 with RawStore

use of org.apache.hadoop.hive.metastore.RawStore in project hive by apache.

the class TestHBaseImport method parallel.

@Test
public void parallel() throws Exception {
    int parallelFactor = 10;
    RawStore rdbms;
    rdbms = new ObjectStore();
    rdbms.setConf(conf);
    String[] dbNames = new String[] { "paralleldb1" };
    int now = (int) System.currentTimeMillis() / 1000;
    for (int i = 0; i < dbNames.length; i++) {
        rdbms.createDatabase(new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
        List<FieldSchema> cols = new ArrayList<>();
        cols.add(new FieldSchema("col1", "int", "nocomment"));
        SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
        StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
        List<FieldSchema> partCols = new ArrayList<>();
        partCols.add(new FieldSchema("region", "string", ""));
        for (int j = 0; j < parallelFactor; j++) {
            rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols, emptyParameters, null, null, null));
            for (int k = 0; k < parallelFactor; k++) {
                StorageDescriptor psd = new StorageDescriptor(sd);
                psd.setLocation("file:/tmp/region=" + k);
                Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j, now, now, psd, emptyParameters);
                rdbms.addPartition(part);
            }
        }
    }
    HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
    importer.setConnections(rdbms, store);
    importer.run();
    for (int i = 0; i < dbNames.length; i++) {
        Database db = store.getDatabase(dbNames[i]);
        Assert.assertNotNull(db);
        for (int j = 0; j < parallelFactor; j++) {
            Table table = store.getTable(db.getName(), "t" + j);
            Assert.assertNotNull(table);
            Assert.assertEquals(now, table.getLastAccessTime());
            Assert.assertEquals("input", table.getSd().getInputFormat());
            for (int k = 0; k < parallelFactor; k++) {
                Partition part = store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
                Assert.assertNotNull(part);
                Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
            }
            Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
        }
        Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
    }
}