Examples with PigServer org.apache.pig.PigServer used on opensource projects

Example 11 with PigServer

use of org.apache.pig.PigServer in project hive by apache.

the class TestHCatStorerMulti method testStorePartitionedTable.

@Test
public void testStorePartitionedTable() throws Exception {
    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
    populateBasicFile();
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);");
    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');");
    server.executeBatch();
    driver.run("select * from " + PARTITIONED_TABLE);
    ArrayList<String> partitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(partitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(), partitionedTableValuesReadFromHiveDriver.size());
}

Example 12 with PigServer

use of org.apache.pig.PigServer in project hive by apache.

the class HCatBaseTest method createPigServer.

/**
 * creates PigServer in LOCAL mode.
 * http://pig.apache.org/docs/r0.12.0/perf.html#error-handling
 * @param stopOnFailure equivalent of "-stop_on_failure" command line arg, setting to 'true' makes
 *                      debugging easier
 */
public static PigServer createPigServer(boolean stopOnFailure) throws ExecException {
    if (stopOnFailure) {
        Properties p = new Properties();
        p.put("stop.on.failure", Boolean.TRUE.toString());
        return new PigServer(ExecType.LOCAL, p);
    }
    return new PigServer(ExecType.LOCAL);
}

Example 13 with PigServer

use of org.apache.pig.PigServer in project hive by apache.

the class TestPassProperties method Initialize.

public void Initialize() throws Exception {
    hiveConf = new HiveConf(this.getClass());
    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
    hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
    driver = DriverFactory.newDriver(hiveConf);
    SessionState.start(new CliSessionState(hiveConf));
    new File(TEST_WAREHOUSE_DIR).mkdirs();
    int numRows = 3;
    input = new String[numRows];
    for (int i = 0; i < numRows; i++) {
        String col1 = "a" + i;
        String col2 = "b" + i;
        input[i] = i + "," + col1 + "," + col2;
    }
    HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input);
    server = new PigServer(ExecType.LOCAL);
}

Example 14 with PigServer

use of org.apache.pig.PigServer in project hive by apache.

the class AbstractHCatLoaderTest method testProjectionsBasic.

@Test
public void testProjectionsBasic() throws IOException {
    PigServer server = createPigServer(false);
    // projections are handled by using generate, not "as" on the Load
    server.registerQuery("Y1 = load '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
    server.registerQuery("Y2 = foreach Y1 generate a;");
    server.registerQuery("Y3 = foreach Y1 generate b,a;");
    Schema dumpedY2Schema = server.dumpSchema("Y2");
    Schema dumpedY3Schema = server.dumpSchema("Y3");
    List<FieldSchema> Y2fields = dumpedY2Schema.getFields();
    List<FieldSchema> Y3fields = dumpedY3Schema.getFields();
    assertEquals(1, Y2fields.size());
    assertEquals("a", Y2fields.get(0).alias.toLowerCase());
    assertEquals(DataType.INTEGER, Y2fields.get(0).type);
    assertEquals(2, Y3fields.size());
    assertEquals("b", Y3fields.get(0).alias.toLowerCase());
    assertEquals(DataType.CHARARRAY, Y3fields.get(0).type);
    assertEquals("a", Y3fields.get(1).alias.toLowerCase());
    assertEquals(DataType.INTEGER, Y3fields.get(1).type);
    int numTuplesRead = 0;
    Iterator<Tuple> Y2Iter = server.openIterator("Y2");
    while (Y2Iter.hasNext()) {
        Tuple t = Y2Iter.next();
        assertEquals(t.size(), 1);
        assertNotNull(t.get(0));
        assertTrue(t.get(0).getClass() == Integer.class);
        assertEquals(t.get(0), basicInputData.get(numTuplesRead).first);
        numTuplesRead++;
    }
    numTuplesRead = 0;
    Iterator<Tuple> Y3Iter = server.openIterator("Y3");
    while (Y3Iter.hasNext()) {
        Tuple t = Y3Iter.next();
        assertEquals(t.size(), 2);
        assertNotNull(t.get(0));
        assertTrue(t.get(0).getClass() == String.class);
        assertEquals(t.get(0), basicInputData.get(numTuplesRead).second);
        assertNotNull(t.get(1));
        assertTrue(t.get(1).getClass() == Integer.class);
        assertEquals(t.get(1), basicInputData.get(numTuplesRead).first);
        numTuplesRead++;
    }
    assertEquals(basicInputData.size(), numTuplesRead);
}

Example 15 with PigServer

use of org.apache.pig.PigServer in project hive by apache.

the class AbstractHCatLoaderTest method testReadMissingPartitionBasicNeg.

@Test
public void testReadMissingPartitionBasicNeg() throws Exception {
    PigServer server = createPigServer(false);
    File removedPartitionDir = new File(TEST_WAREHOUSE_DIR + "/" + PARTITIONED_TABLE + "/bkt=0");
    if (!removeDirectory(removedPartitionDir)) {
        System.out.println("Test did not run because its environment could not be set.");
        return;
    }
    driver.run("select * from " + PARTITIONED_TABLE);
    ArrayList<String> valuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(valuesReadFromHiveDriver);
    assertTrue(valuesReadFromHiveDriver.size() == 6);
    server.registerQuery("W = load '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
    Schema dumpedWSchema = server.dumpSchema("W");
    List<FieldSchema> Wfields = dumpedWSchema.getFields();
    assertEquals(3, Wfields.size());
    assertTrue(Wfields.get(0).alias.equalsIgnoreCase("a"));
    assertTrue(Wfields.get(0).type == DataType.INTEGER);
    assertTrue(Wfields.get(1).alias.equalsIgnoreCase("b"));
    assertTrue(Wfields.get(1).type == DataType.CHARARRAY);
    assertTrue(Wfields.get(2).alias.equalsIgnoreCase("bkt"));
    assertTrue(Wfields.get(2).type == DataType.CHARARRAY);
    try {
        Iterator<Tuple> WIter = server.openIterator("W");
        fail("Should failed in retriving an invalid partition");
    } catch (IOException ioe) {
    // expected
    }
}