Examples with StorageDescriptor org.apache.hadoop.hive.metastore.api.StorageDescriptor used on opensource projects

Example 11 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestHBaseStoreIntegration method alterPartitions.

@Test
public void alterPartitions() throws Exception {
    String dbName = "default";
    String tableName = "alterParts";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema("pc", "string", ""));
    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    store.createTable(table);
    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
    List<Partition> partitions = new ArrayList<Partition>();
    List<List<String>> allVals = new ArrayList<List<String>>();
    for (String val : partVals) {
        List<String> vals = new ArrayList<String>();
        allVals.add(vals);
        vals.add(val);
        StorageDescriptor psd = new StorageDescriptor(sd);
        psd.setLocation("file:/tmp/pc=" + val);
        Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters);
        partitions.add(part);
    }
    store.addPartitions(dbName, tableName, partitions);
    for (Partition p : partitions) p.setLastAccessTime(startTime + 10);
    store.alterPartitions(dbName, tableName, allVals, partitions);
    partitions = store.getPartitions(dbName, tableName, -1);
    for (Partition part : partitions) {
        Assert.assertEquals(startTime + 10, part.getLastAccessTime());
    }
}

Example 12 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestHBaseStoreIntegration method getPartitions.

@Test
public void getPartitions() throws Exception {
    String dbName = "default";
    String tableName = "manyParts";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema("pc", "string", ""));
    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    store.createTable(table);
    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
    for (String val : partVals) {
        List<String> vals = new ArrayList<String>();
        vals.add(val);
        StorageDescriptor psd = new StorageDescriptor(sd);
        psd.setLocation("file:/tmp/pc=" + val);
        Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters);
        store.addPartition(part);
        Partition p = store.getPartition(dbName, tableName, vals);
        Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
    }
    List<Partition> parts = store.getPartitions(dbName, tableName, -1);
    Assert.assertEquals(5, parts.size());
    String[] pv = new String[5];
    for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
    Arrays.sort(pv);
    Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
}

Example 13 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestHBaseStoreIntegration method addPartitions.

@Test
public void addPartitions() throws Exception {
    String dbName = "default";
    String tableName = "addParts";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema("pc", "string", ""));
    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    store.createTable(table);
    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
    List<Partition> partitions = new ArrayList<Partition>();
    for (String val : partVals) {
        List<String> vals = new ArrayList<String>();
        vals.add(val);
        StorageDescriptor psd = new StorageDescriptor(sd);
        psd.setLocation("file:/tmp/pc=" + val);
        Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters);
        partitions.add(part);
    }
    store.addPartitions(dbName, tableName, partitions);
    List<String> partNames = store.listPartitionNames(dbName, tableName, (short) -1);
    Assert.assertEquals(5, partNames.size());
    String[] names = partNames.toArray(new String[partNames.size()]);
    Arrays.sort(names);
    String[] canonicalNames = partVals.toArray(new String[partVals.size()]);
    for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i];
    Assert.assertArrayEquals(canonicalNames, names);
}

Example 14 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestHBaseStoreIntegration method listTableGrants.

@Test
public void listTableGrants() throws Exception {
    String dbName = "ltg_db";
    String[] tableNames = new String[] { "ltg_t1", "ltg_t2" };
    try {
        Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
        store.createDatabase(db);
        int startTime = (int) (System.currentTimeMillis() / 1000);
        List<FieldSchema> cols = new ArrayList<FieldSchema>();
        cols.add(new FieldSchema("col1", "int", "nocomment"));
        SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
        StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
        Table table = new Table(tableNames[0], dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
        store.createTable(table);
        table = new Table(tableNames[1], dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
        store.createTable(table);
        String[] roleNames = new String[] { "ltg_role1", "ltg_role2" };
        String[] userNames = new String[] { "gandalf", "radagast" };
        store.addRole(roleNames[0], "me");
        store.addRole(roleNames[1], "me");
        int now = (int) (System.currentTimeMillis() / 1000);
        Role role1 = store.getRole(roleNames[0]);
        Role role2 = store.getRole(roleNames[1]);
        store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
        store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
        store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
        List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
        HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.TABLE, dbName, tableNames[0], null, null);
        PrivilegeGrantInfo grantInfo = new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
        HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER, grantInfo);
        privileges.add(hop);
        grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
        hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
        privileges.add(hop);
        PrivilegeBag pBag = new PrivilegeBag(privileges);
        store.grantPrivileges(pBag);
        List<HiveObjectPrivilege> hops = store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[0]);
        Assert.assertEquals(1, hops.size());
        Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
        Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
        Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
        hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[0]);
        Assert.assertEquals(1, hops.size());
        Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
        Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
        Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
        hops = store.listAllTableGrants(roleNames[1], PrincipalType.ROLE, dbName, tableNames[0]);
        Assert.assertEquals(0, hops.size());
        hops = store.listAllTableGrants(userNames[1], PrincipalType.USER, dbName, tableNames[0]);
        Assert.assertEquals(0, hops.size());
        hops = store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[1]);
        Assert.assertEquals(0, hops.size());
        hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[1]);
        Assert.assertEquals(0, hops.size());
        hops = store.listTableGrantsAll(dbName, tableNames[0]);
        Assert.assertEquals(2, hops.size());
        boolean sawUser = false, sawRole = false;
        for (HiveObjectPrivilege h : hops) {
            if (h.getPrincipalName().equals(userNames[0])) {
                Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
                Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType());
                Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
                sawUser = true;
            } else if (h.getPrincipalName().equals(roleNames[0])) {
                Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
                Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType());
                Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
                sawRole = true;
            }
        }
        Assert.assertTrue(sawUser && sawRole);
        hops = store.listPrincipalTableGrantsAll(roleNames[0], PrincipalType.ROLE);
        Assert.assertEquals(1, hops.size());
        Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
        Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
        Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
        hops = store.listPrincipalTableGrantsAll(userNames[0], PrincipalType.USER);
        Assert.assertEquals(1, hops.size());
        Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
        Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
        Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
        hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE);
        Assert.assertEquals(0, hops.size());
        hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER);
        Assert.assertEquals(0, hops.size());
    } finally {
        store.dropTable(dbName, tableNames[0]);
        store.dropTable(dbName, tableNames[1]);
        store.dropDatabase(dbName);
    }
}

Example 15 with StorageDescriptor

use of org.apache.hadoop.hive.metastore.api.StorageDescriptor in project hive by apache.

the class TestHBaseImport method parallelOdd.

// Same as the test above except we create 9 of everything instead of 10.  This is important
// because in using a batch size of 2 the previous test guarantees 10 /2 =5 , meaning we'll
// have 5 writes on the partition queue with exactly 2 entries.  In this test we'll handle the
// case where the last entry in the queue has fewer partitions.
@Test
public void parallelOdd() throws Exception {
    int parallelFactor = 9;
    RawStore rdbms;
    rdbms = new ObjectStore();
    rdbms.setConf(conf);
    String[] dbNames = new String[] { "oddparalleldb1" };
    int now = (int) System.currentTimeMillis() / 1000;
    for (int i = 0; i < dbNames.length; i++) {
        rdbms.createDatabase(new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
        List<FieldSchema> cols = new ArrayList<>();
        cols.add(new FieldSchema("col1", "int", "nocomment"));
        SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
        StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
        List<FieldSchema> partCols = new ArrayList<>();
        partCols.add(new FieldSchema("region", "string", ""));
        for (int j = 0; j < parallelFactor; j++) {
            rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols, emptyParameters, null, null, null));
            for (int k = 0; k < parallelFactor; k++) {
                StorageDescriptor psd = new StorageDescriptor(sd);
                psd.setLocation("file:/tmp/region=" + k);
                Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j, now, now, psd, emptyParameters);
                rdbms.addPartition(part);
            }
        }
    }
    HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
    importer.setConnections(rdbms, store);
    importer.run();
    for (int i = 0; i < dbNames.length; i++) {
        Database db = store.getDatabase(dbNames[i]);
        Assert.assertNotNull(db);
        for (int j = 0; j < parallelFactor; j++) {
            Table table = store.getTable(db.getName(), "t" + j);
            Assert.assertNotNull(table);
            Assert.assertEquals(now, table.getLastAccessTime());
            Assert.assertEquals("input", table.getSd().getInputFormat());
            for (int k = 0; k < parallelFactor; k++) {
                Partition part = store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
                Assert.assertNotNull(part);
                Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
            }
            Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
        }
        Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
    }
}