Examples with Data org.apache.pig.builtin.mock.Storage.Data used on opensource projects

Example 1 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testLoadForArrayWithTable.

/**
     * 
     * @throws Exception
     */
@Test
public void testLoadForArrayWithTable() throws Exception {
    //create the table
    final String TABLE = "TABLE15";
    String ddl = "CREATE TABLE  " + TABLE + " ( ID INTEGER PRIMARY KEY, a_double_array double array[])";
    conn.createStatement().execute(ddl);
    Double[] doubleArr = new Double[3];
    doubleArr[0] = 2.2;
    doubleArr[1] = 4.4;
    doubleArr[2] = 6.6;
    Array doubleArray = conn.createArrayOf("DOUBLE", doubleArr);
    Tuple doubleArrTuple = Storage.tuple(2.2d, 4.4d, 6.6d);
    Double[] doubleArr2 = new Double[2];
    doubleArr2[0] = 12.2;
    doubleArr2[1] = 22.2;
    Array doubleArray2 = conn.createArrayOf("DOUBLE", doubleArr2);
    Tuple doubleArrTuple2 = Storage.tuple(12.2d, 22.2d);
    //upsert data.
    final String dml = "UPSERT INTO " + TABLE + " VALUES(?, ?) ";
    PreparedStatement stmt = conn.prepareStatement(dml);
    stmt.setInt(1, 1);
    stmt.setArray(2, doubleArray);
    stmt.execute();
    stmt.setInt(1, 2);
    stmt.setArray(2, doubleArray2);
    stmt.execute();
    conn.commit();
    final Data data = Storage.resetData(pigServer);
    List<Tuple> expectedList = new ArrayList<Tuple>();
    expectedList.add(Storage.tuple(1, doubleArrTuple));
    expectedList.add(Storage.tuple(2, doubleArrTuple2));
    pigServer.setBatchOn();
    pigServer.registerQuery(String.format("A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", TABLE, zkQuorum));
    pigServer.registerQuery("STORE A INTO 'out' using mock.Storage();");
    pigServer.executeBatch();
    List<Tuple> actualList = data.get("out");
    assertEquals(expectedList.size(), actualList.size());
    assertEquals(expectedList, actualList);
}

Example 2 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testLoadOfSaltTable.

@Test
public void testLoadOfSaltTable() throws Exception {
    final String TABLE = "TABLE11";
    final String sourceTableddl = "CREATE TABLE  " + TABLE + "  (ID  INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER, SAL INTEGER) SALT_BUCKETS=2  ";
    conn.createStatement().execute(sourceTableddl);
    //prepare data with 10 rows having age 25 and the other 30.
    final String dml = "UPSERT INTO " + TABLE + " VALUES(?,?,?,?)";
    PreparedStatement stmt = conn.prepareStatement(dml);
    int rows = 20;
    int j = 0, k = 0;
    for (int i = 0; i < rows; i++) {
        stmt.setInt(1, i);
        stmt.setString(2, "a" + i);
        if (i % 2 == 0) {
            stmt.setInt(3, 25);
            stmt.setInt(4, 10 * 2 * j++);
        } else {
            stmt.setInt(3, 30);
            stmt.setInt(4, 10 * 3 * k++);
        }
        stmt.execute();
    }
    conn.commit();
    final Data data = Storage.resetData(pigServer);
    List<Tuple> expectedList = new ArrayList<Tuple>();
    expectedList.add(Storage.tuple(25, 10));
    expectedList.add(Storage.tuple(30, 10));
    pigServer.setBatchOn();
    pigServer.registerQuery(String.format("A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", TABLE, zkQuorum));
    pigServer.registerQuery("B = GROUP A BY AGE;");
    pigServer.registerQuery("C = FOREACH B GENERATE group,COUNT(A);");
    pigServer.registerQuery("STORE C INTO 'out' using mock.Storage();");
    pigServer.executeBatch();
    List<Tuple> actualList = data.get("out");
    assertEquals(expectedList.size(), actualList.size());
}

Example 3 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseStorerIT method testStorerForSpecificColumns.

/**
     * Basic test - writes specific columns data to a Phoenix table and compares the data written
     * to expected
     * 
     * @throws Exception
     */
@Test
public void testStorerForSpecificColumns() throws Exception {
    final String tableName = SchemaUtil.getTableName("TABLE2", SchemaUtil.getEscapedArgument("zo2"));
    final Statement stmt = conn.createStatement();
    stmt.execute("CREATE TABLE " + tableName + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER)");
    final Data data = Storage.resetData(pigServer);
    final Collection<Tuple> list = Lists.newArrayList();
    // Create input dataset
    int rows = 100;
    for (int i = 0; i < rows; i++) {
        Tuple t = tupleFactory.newTuple();
        t.append(i);
        t.append("a" + i);
        t.append(i * 2);
        list.add(t);
    }
    data.set("in", "id:int, name:chararray,age:int", list);
    pigServer.setBatchOn();
    pigServer.registerQuery("A = LOAD 'in' USING mock.Storage();");
    pigServer.registerQuery("B = FOREACH A GENERATE id,name;");
    pigServer.registerQuery("Store B into 'hbase://" + tableName + "/ID,NAME" + "' using " + PhoenixHBaseStorage.class.getName() + "('" + zkQuorum + "', '-batchSize 1000');");
    // Now run the Pig script
    if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) {
        throw new RuntimeException("Job failed", pigServer.executeBatch().get(0).getException());
    }
    // Compare data in Phoenix table to the expected
    final ResultSet rs = stmt.executeQuery("SELECT id, name,age FROM " + tableName + " ORDER BY id");
    for (int i = 0; i < rows; i++) {
        assertTrue(rs.next());
        assertEquals(i, rs.getInt(1));
        assertEquals("a" + i, rs.getString(2));
        assertEquals(0, rs.getInt(3));
    }
}

Example 4 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testGroupingOfDataForTable.

/**
     * @throws Exception
     */
@Test
public void testGroupingOfDataForTable() throws Exception {
    //create the table
    final String TABLE = "TABLE6";
    String ddl = "CREATE TABLE  " + TABLE + "  (ID  INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER, SAL INTEGER) ";
    conn.createStatement().execute(ddl);
    //prepare data with 10 rows having age 25 and the other 30.
    final String dml = "UPSERT INTO " + TABLE + " VALUES(?,?,?,?)";
    PreparedStatement stmt = conn.prepareStatement(dml);
    int rows = 20;
    int j = 0, k = 0;
    for (int i = 0; i < rows; i++) {
        stmt.setInt(1, i);
        stmt.setString(2, "a" + i);
        if (i % 2 == 0) {
            stmt.setInt(3, 25);
            stmt.setInt(4, 10 * 2 * j++);
        } else {
            stmt.setInt(3, 30);
            stmt.setInt(4, 10 * 3 * k++);
        }
        stmt.execute();
    }
    conn.commit();
    //prepare the mock storage with expected output
    final Data data = Storage.resetData(pigServer);
    List<Tuple> expectedList = new ArrayList<Tuple>();
    expectedList.add(Storage.tuple(0, 180));
    expectedList.add(Storage.tuple(0, 270));
    //load data and filter rows whose age is > 25
    pigServer.setBatchOn();
    pigServer.registerQuery(String.format("A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", TABLE, zkQuorum));
    pigServer.registerQuery("B = GROUP A BY AGE;");
    pigServer.registerQuery("C = FOREACH B GENERATE MIN(A.SAL),MAX(A.SAL);");
    pigServer.registerQuery("STORE C INTO 'out' using mock.Storage();");
    pigServer.executeBatch();
    List<Tuple> actualList = data.get("out");
    assertEquals(expectedList, actualList);
}

Example 5 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseStorerIT method testStorer.

/**
     * Basic test - writes data to a Phoenix table and compares the data written
     * to expected
     * 
     * @throws Exception
     */
@Test
public void testStorer() throws Exception {
    final String tableName = "TABLE1";
    final Statement stmt = conn.createStatement();
    stmt.execute("CREATE TABLE " + tableName + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR)");
    final Data data = Storage.resetData(pigServer);
    final Collection<Tuple> list = Lists.newArrayList();
    // Create input dataset
    int rows = 100;
    for (int i = 0; i < rows; i++) {
        Tuple t = tupleFactory.newTuple();
        t.append(i);
        t.append("a" + i);
        list.add(t);
    }
    data.set("in", "id:int, name:chararray", list);
    pigServer.setBatchOn();
    pigServer.registerQuery("A = LOAD 'in' USING mock.Storage();");
    pigServer.registerQuery("Store A into 'hbase://" + tableName + "' using " + PhoenixHBaseStorage.class.getName() + "('" + zkQuorum + "', '-batchSize 1000');");
    // Now run the Pig script
    if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) {
        throw new RuntimeException("Job failed", pigServer.executeBatch().get(0).getException());
    }
    // Compare data in Phoenix table to the expected
    final ResultSet rs = stmt.executeQuery("SELECT id, name FROM table1 ORDER BY id");
    for (int i = 0; i < rows; i++) {
        assertTrue(rs.next());
        assertEquals(i, rs.getInt(1));
        assertEquals("a" + i, rs.getString(2));
    }
}