Examples with Pair org.apache.hive.hcatalog.data.Pair used on opensource projects

Example 6 with Pair

use of org.apache.hive.hcatalog.data.Pair in project hive by apache.

the class AbstractHCatLoaderTest method setUpTest.

@Before
public void setUpTest() throws Exception {
    createTableDefaultDB(BASIC_TABLE, "a int, b string");
    createTableDefaultDB(COMPLEX_TABLE, "name string, studentid int, " + "contact struct<phno:string,email:string>, " + "currently_registered_courses array<string>, " + "current_grades map<string,string>, " + "phnos array<struct<phno:string,type:string>>");
    createTableDefaultDB(PARTITIONED_TABLE, "a int, b string", "bkt string");
    createTableDefaultDB(SPECIFIC_SIZE_TABLE, "a int, b string");
    createTable(SPECIFIC_DATABASE, SPECIFIC_SIZE_TABLE_2, "a int, b string");
    createTableDefaultDB(PARTITIONED_DATE_TABLE, "b string", "dt date");
    AllTypesTable.setupAllTypesTable(driver);
    int LOOP_SIZE = 3;
    String[] input = new String[LOOP_SIZE * LOOP_SIZE];
    basicInputData = new HashMap<Integer, Pair<Integer, String>>();
    int k = 0;
    for (int i = 1; i <= LOOP_SIZE; i++) {
        String si = i + "";
        for (int j = 1; j <= LOOP_SIZE; j++) {
            String sj = "S" + j + "S";
            input[k] = si + "\t" + sj;
            basicInputData.put(k, new Pair<Integer, String>(i, sj));
            k++;
        }
    }
    HcatTestUtils.createTestDataFile(BASIC_FILE_NAME, input);
    HcatTestUtils.createTestDataFile(COMPLEX_FILE_NAME, new String[] { "Henry Jekyll\t42\t(415-253-6367,hjekyll@contemporary.edu.uk)\t{(PHARMACOLOGY),(PSYCHIATRY)}\t[PHARMACOLOGY#A-,PSYCHIATRY#B+]\t{(415-253-6367,cell),(408-253-6367,landline)}", "Edward Hyde\t1337\t(415-253-6367,anonymous@b44chan.org)\t{(CREATIVE_WRITING),(COPYRIGHT_LAW)}\t[CREATIVE_WRITING#A+,COPYRIGHT_LAW#D]\t{(415-253-6367,cell),(408-253-6367,landline)}" });
    HcatTestUtils.createTestDataFile(DATE_FILE_NAME, new String[] { "2016-07-14 08:10:15\tHenry Jekyll", "2016-07-15 11:54:55\tEdward Hyde" });
    PigServer server = createPigServer(false);
    server.setBatchOn();
    int i = 0;
    server.registerQuery("A = load '" + BASIC_FILE_NAME + "' as (a:int, b:chararray);", ++i);
    server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
    server.registerQuery("store A into '" + SPECIFIC_SIZE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
    server.registerQuery("store A into '" + SPECIFIC_DATABASE + "." + SPECIFIC_SIZE_TABLE_2 + "' " + "using org.apache.hive" + ".hcatalog.pig.HCatStorer();", ++i);
    server.registerQuery("B = foreach A generate a,b;", ++i);
    server.registerQuery("B2 = filter B by a < 2;", ++i);
    server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');", ++i);
    server.registerQuery("C = foreach A generate a,b;", ++i);
    server.registerQuery("C2 = filter C by a >= 2;", ++i);
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');", ++i);
    server.registerQuery("D = load '" + COMPLEX_FILE_NAME + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});", ++i);
    server.registerQuery("store D into '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
    server.registerQuery("E = load '" + DATE_FILE_NAME + "' as (dt:chararray, b:chararray);", ++i);
    server.registerQuery("F = foreach E generate ToDate(dt, 'yyyy-MM-dd HH:mm:ss') as dt, b;", ++i);
    server.registerQuery("store F into '" + PARTITIONED_DATE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
    server.executeBatch();
}