Examples with Data org.apache.pig.builtin.mock.Storage.Data used on opensource projects

Example 6 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseStorerIT method testStoreWithBinaryDataTypes.

/**
     * Test storage of DataByteArray columns to Phoenix
     * Maps the DataByteArray with the target PhoenixDataType and persists in HBase. 
    * @throws Exception
     */
@Test
public void testStoreWithBinaryDataTypes() throws Exception {
    final String tableName = "TABLE3";
    final Statement stmt = conn.createStatement();
    stmt.execute("CREATE TABLE " + tableName + " (col1 BIGINT NOT NULL, col2 INTEGER , col3 FLOAT, col4 DOUBLE , col5 TINYINT , " + "  col6 BOOLEAN , col7 VARBINARY CONSTRAINT my_pk PRIMARY KEY (col1))");
    final Data data = Storage.resetData(pigServer);
    final Collection<Tuple> list = Lists.newArrayList();
    int rows = 10;
    for (int i = 1; i <= rows; i++) {
        Tuple t = tupleFactory.newTuple();
        t.append(i);
        t.append(new DataByteArray(Bytes.toBytes(i * 5)));
        t.append(new DataByteArray(Bytes.toBytes(i * 10.0F)));
        t.append(new DataByteArray(Bytes.toBytes(i * 15.0D)));
        t.append(new DataByteArray(Bytes.toBytes(i)));
        t.append(new DataByteArray(Bytes.toBytes(i % 2 == 0)));
        t.append(new DataByteArray(Bytes.toBytes(i)));
        list.add(t);
    }
    data.set("in", "col1:int,col2:bytearray,col3:bytearray,col4:bytearray,col5:bytearray,col6:bytearray,col7:bytearray ", list);
    pigServer.setBatchOn();
    pigServer.registerQuery("A = LOAD 'in' USING mock.Storage();");
    pigServer.registerQuery("Store A into 'hbase://" + tableName + "' using " + PhoenixHBaseStorage.class.getName() + "('" + zkQuorum + "', '-batchSize 1000');");
    if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) {
        throw new RuntimeException("Job failed", pigServer.executeBatch().get(0).getException());
    }
    final ResultSet rs = stmt.executeQuery(String.format("SELECT col1 , col2 , col3 , col4 , col5 , col6, col7  FROM %s ORDER BY col1", tableName));
    int count = 0;
    for (int i = 1; i <= rows; i++) {
        assertTrue(rs.next());
        assertEquals(i, rs.getInt(1));
        assertEquals(i * 5, rs.getInt(2));
        assertEquals(i * 10.0F, rs.getFloat(3), 0.0);
        assertEquals(i * 15.0D, rs.getInt(4), 0.0);
        assertEquals(i, rs.getInt(5));
        assertEquals(i % 2 == 0, rs.getBoolean(6));
        assertArrayEquals(Bytes.toBytes(i), rs.getBytes(7));
        count++;
    }
    assertEquals(rows, count);
}

Example 7 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseStorerIT method testStoreForArray.

@Test
public void testStoreForArray() throws Exception {
    final String tableName = "TABLE5";
    final Statement stmt = conn.createStatement();
    String ddl = "CREATE TABLE  " + tableName + " ( ID INTEGER PRIMARY KEY, dbl double array[], a_varchar_array varchar array)";
    stmt.execute(ddl);
    final Data data = Storage.resetData(pigServer);
    data.set("in", tuple(1, tuple(2.2)), tuple(2, tuple(2.4, 2.5)), tuple(3, tuple(2.3)));
    pigServer.setBatchOn();
    pigServer.registerQuery("A = LOAD 'in' USING mock.Storage() as (id:int, dbl:tuple());");
    pigServer.registerQuery("Store A into 'hbase://" + tableName + "/ID,DBL" + "' using " + PhoenixHBaseStorage.class.getName() + "('" + zkQuorum + "', '-batchSize 1000');");
    if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) {
        throw new RuntimeException("Job failed", pigServer.executeBatch().get(0).getException());
    }
    final ResultSet rs = stmt.executeQuery(String.format("SELECT id , dbl FROM %s where id = 2", tableName));
    assertTrue(rs.next());
    assertEquals(2, rs.getInt(1));
    Array expectedDoubleArr = conn.createArrayOf("DOUBLE", new Double[] { 2.4, 2.5 });
    assertEquals(expectedDoubleArr, rs.getArray(2));
}

Example 8 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseStorerIT method testStoreWithDateTime.

@Test
public void testStoreWithDateTime() throws Exception {
    final String tableName = "TABLE4";
    final Statement stmt = conn.createStatement();
    stmt.execute("CREATE TABLE " + tableName + " (col1 BIGINT NOT NULL, col2 DATE , col3 TIME, " + " col4 TIMESTAMP CONSTRAINT my_pk PRIMARY KEY (col1))");
    long now = System.currentTimeMillis();
    final DateTime dt = new DateTime(now);
    final Data data = Storage.resetData(pigServer);
    final Collection<Tuple> list = Lists.newArrayList();
    Tuple t = tupleFactory.newTuple();
    t.append(1);
    t.append(dt);
    t.append(dt);
    t.append(dt);
    list.add(t);
    data.set("in", "col1:int,col2:datetime,col3:datetime,col4:datetime", list);
    pigServer.setBatchOn();
    pigServer.registerQuery("A = LOAD 'in' USING mock.Storage();");
    pigServer.registerQuery("Store A into 'hbase://" + tableName + "' using " + PhoenixHBaseStorage.class.getName() + "('" + zkQuorum + "', '-batchSize 1000');");
    if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) {
        throw new RuntimeException("Job failed", pigServer.executeBatch().get(0).getException());
    }
    final ResultSet rs = stmt.executeQuery(String.format("SELECT col1 , col2 , col3 , col4 FROM %s ", tableName));
    assertTrue(rs.next());
    assertEquals(1, rs.getInt(1));
    assertEquals(now, rs.getDate(2).getTime());
    assertEquals(now, rs.getTime(3).getTime());
    assertEquals(now, rs.getTimestamp(4).getTime());
}

Example 9 with Data

use of org.apache.pig.builtin.mock.Storage.Data in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testLoadForArrayWithQuery.

/**
    * 
    * @throws Exception
    */
@Test
public void testLoadForArrayWithQuery() throws Exception {
    //create the table
    final String TABLE = "TABLE14";
    String ddl = "CREATE TABLE  " + TABLE + " ( ID INTEGER PRIMARY KEY, a_double_array double array[] , a_varchar_array varchar array, a_concat_str varchar, sep varchar)";
    conn.createStatement().execute(ddl);
    Double[] doubleArr = new Double[3];
    doubleArr[0] = 2.2;
    doubleArr[1] = 4.4;
    doubleArr[2] = 6.6;
    Array doubleArray = conn.createArrayOf("DOUBLE", doubleArr);
    Tuple doubleArrTuple = Storage.tuple(2.2d, 4.4d, 6.6d);
    Double[] doubleArr2 = new Double[2];
    doubleArr2[0] = 12.2;
    doubleArr2[1] = 22.2;
    Array doubleArray2 = conn.createArrayOf("DOUBLE", doubleArr2);
    Tuple doubleArrTuple2 = Storage.tuple(12.2d, 22.2d);
    String[] strArr = new String[4];
    strArr[0] = "ABC";
    strArr[1] = "DEF";
    strArr[2] = "GHI";
    strArr[3] = "JKL";
    Array strArray = conn.createArrayOf("VARCHAR", strArr);
    Tuple strArrTuple = Storage.tuple("ABC", "DEF", "GHI", "JKL");
    String[] strArr2 = new String[2];
    strArr2[0] = "ABC";
    strArr2[1] = "XYZ";
    Array strArray2 = conn.createArrayOf("VARCHAR", strArr2);
    Tuple strArrTuple2 = Storage.tuple("ABC", "XYZ");
    //upsert data.
    final String dml = "UPSERT INTO " + TABLE + " VALUES(?, ?, ?, ?, ?) ";
    PreparedStatement stmt = conn.prepareStatement(dml);
    stmt.setInt(1, 1);
    stmt.setArray(2, doubleArray);
    stmt.setArray(3, strArray);
    stmt.setString(4, "ONE,TWO,THREE");
    stmt.setString(5, ",");
    stmt.execute();
    stmt.setInt(1, 2);
    stmt.setArray(2, doubleArray2);
    stmt.setArray(3, strArray2);
    stmt.setString(4, "FOUR:five:six");
    stmt.setString(5, ":");
    stmt.execute();
    conn.commit();
    Tuple dynArrTuple = Storage.tuple("ONE", "TWO", "THREE");
    Tuple dynArrTuple2 = Storage.tuple("FOUR", "five", "six");
    //sql query
    final String sqlQuery = String.format(" SELECT ID, A_DOUBLE_ARRAY, A_VARCHAR_ARRAY, REGEXP_SPLIT(a_concat_str, sep) AS flattend_str FROM %s ", TABLE);
    final Data data = Storage.resetData(pigServer);
    List<Tuple> expectedList = new ArrayList<Tuple>();
    expectedList.add(Storage.tuple(1, 3L, 4L, dynArrTuple));
    expectedList.add(Storage.tuple(2, 2L, 2L, dynArrTuple2));
    final String load = String.format("A = load 'hbase://query/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", sqlQuery, zkQuorum);
    pigServer.setBatchOn();
    pigServer.registerQuery(load);
    pigServer.registerQuery("B = FOREACH A GENERATE ID, SIZE(A_DOUBLE_ARRAY), SIZE(A_VARCHAR_ARRAY), FLATTEND_STR;");
    pigServer.registerQuery("STORE B INTO 'out' using mock.Storage();");
    pigServer.executeBatch();
    List<Tuple> actualList = data.get("out");
    assertEquals(expectedList.size(), actualList.size());
    assertEquals(expectedList, actualList);
    Schema schema = pigServer.dumpSchema("A");
    List<FieldSchema> fields = schema.getFields();
    assertEquals(4, fields.size());
    assertTrue(fields.get(0).alias.equalsIgnoreCase("ID"));
    assertTrue(fields.get(0).type == DataType.INTEGER);
    assertTrue(fields.get(1).alias.equalsIgnoreCase("A_DOUBLE_ARRAY"));
    assertTrue(fields.get(1).type == DataType.TUPLE);
    assertTrue(fields.get(2).alias.equalsIgnoreCase("A_VARCHAR_ARRAY"));
    assertTrue(fields.get(2).type == DataType.TUPLE);
    assertTrue(fields.get(3).alias.equalsIgnoreCase("FLATTEND_STR"));
    assertTrue(fields.get(3).type == DataType.TUPLE);
    Iterator<Tuple> iterator = pigServer.openIterator("A");
    Tuple firstTuple = Storage.tuple(1, doubleArrTuple, strArrTuple, dynArrTuple);
    Tuple secondTuple = Storage.tuple(2, doubleArrTuple2, strArrTuple2, dynArrTuple2);
    List<Tuple> expectedRows = Lists.newArrayList(firstTuple, secondTuple);
    List<Tuple> actualRows = Lists.newArrayList();
    while (iterator.hasNext()) {
        Tuple tuple = iterator.next();
        actualRows.add(tuple);
    }
    assertEquals(expectedRows, actualRows);
}