Examples with FieldSchema org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema used on opensource projects

Example 11 with FieldSchema

use of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testSchemaForTable.

/**
     * Validates the schema returned for a table with Pig data types.
     * @throws Exception
     */
@Test
public void testSchemaForTable() throws Exception {
    final String TABLE = "TABLE1";
    final String ddl = String.format("CREATE TABLE %s " + "  (a_string varchar not null, a_binary varbinary not null, a_integer integer, cf1.a_float float" + "  CONSTRAINT pk PRIMARY KEY (a_string, a_binary))\n", TABLE);
    conn.createStatement().execute(ddl);
    conn.commit();
    pigServer.registerQuery(String.format("A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", TABLE, zkQuorum));
    final Schema schema = pigServer.dumpSchema("A");
    List<FieldSchema> fields = schema.getFields();
    assertEquals(4, fields.size());
    assertTrue(fields.get(0).alias.equalsIgnoreCase("a_string"));
    assertTrue(fields.get(0).type == DataType.CHARARRAY);
    assertTrue(fields.get(1).alias.equalsIgnoreCase("a_binary"));
    assertTrue(fields.get(1).type == DataType.BYTEARRAY);
    assertTrue(fields.get(2).alias.equalsIgnoreCase("a_integer"));
    assertTrue(fields.get(2).type == DataType.INTEGER);
    assertTrue(fields.get(3).alias.equalsIgnoreCase("a_float"));
    assertTrue(fields.get(3).type == DataType.FLOAT);
}

Example 12 with FieldSchema

use of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testForNonPKSQLQuery.

/**
     * 
     * @throws Exception
     */
@Test
public void testForNonPKSQLQuery() throws Exception {
    //create the table
    final String TABLE = "TABLE5";
    String ddl = "CREATE TABLE  " + TABLE + " ( ID VARCHAR PRIMARY KEY, FOO VARCHAR, BAR INTEGER, BAZ UNSIGNED_INT)";
    conn.createStatement().execute(ddl);
    //upsert data.
    final String dml = "UPSERT INTO " + TABLE + " VALUES(?,?,?,?) ";
    PreparedStatement stmt = conn.prepareStatement(dml);
    stmt.setString(1, "a");
    stmt.setString(2, "a");
    stmt.setInt(3, -1);
    stmt.setInt(4, 1);
    stmt.execute();
    stmt.setString(1, "b");
    stmt.setString(2, "b");
    stmt.setInt(3, -2);
    stmt.setInt(4, 2);
    stmt.execute();
    conn.commit();
    //sql query
    final String sqlQuery = String.format(" SELECT FOO, BAZ FROM %s WHERE BAR = -1 ", TABLE);
    pigServer.registerQuery(String.format("A = load 'hbase://query/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", sqlQuery, zkQuorum));
    final Iterator<Tuple> iterator = pigServer.openIterator("A");
    int recordsRead = 0;
    while (iterator.hasNext()) {
        final Tuple tuple = iterator.next();
        assertEquals("a", tuple.get(0));
        assertEquals(1, tuple.get(1));
        recordsRead++;
    }
    assertEquals(1, recordsRead);
    //test the schema. Test for PHOENIX-1123
    Schema schema = pigServer.dumpSchema("A");
    List<FieldSchema> fields = schema.getFields();
    assertEquals(2, fields.size());
    assertTrue(fields.get(0).alias.equalsIgnoreCase("FOO"));
    assertTrue(fields.get(0).type == DataType.CHARARRAY);
    assertTrue(fields.get(1).alias.equalsIgnoreCase("BAZ"));
    assertTrue(fields.get(1).type == DataType.INTEGER);
}

Example 13 with FieldSchema

use of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testSchemaForQuery.

/**
     * Validates the schema returned when a SQL SELECT query is given as part of LOAD .
     * @throws Exception
     */
@Test
public void testSchemaForQuery() throws Exception {
    //create the table.
    final String TABLE = "TABLE3";
    String ddl = String.format("CREATE TABLE " + TABLE + "  (A_STRING VARCHAR NOT NULL, A_DECIMAL DECIMAL NOT NULL, CF1.A_INTEGER INTEGER, CF2.A_DOUBLE DOUBLE" + "  CONSTRAINT pk PRIMARY KEY (A_STRING, A_DECIMAL))\n", TABLE);
    conn.createStatement().execute(ddl);
    //sql query for LOAD
    final String sqlQuery = "SELECT A_STRING,CF1.A_INTEGER,CF2.A_DOUBLE FROM " + TABLE;
    pigServer.registerQuery(String.format("A = load 'hbase://query/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", sqlQuery, zkQuorum));
    //assert the schema.
    Schema schema = pigServer.dumpSchema("A");
    List<FieldSchema> fields = schema.getFields();
    assertEquals(3, fields.size());
    assertTrue(fields.get(0).alias.equalsIgnoreCase("a_string"));
    assertTrue(fields.get(0).type == DataType.CHARARRAY);
    assertTrue(fields.get(1).alias.equalsIgnoreCase("a_integer"));
    assertTrue(fields.get(1).type == DataType.INTEGER);
    assertTrue(fields.get(2).alias.equalsIgnoreCase("a_double"));
    assertTrue(fields.get(2).type == DataType.DOUBLE);
}

Example 14 with FieldSchema

use of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema in project phoenix by apache.

the class PhoenixHBaseLoaderIT method testLoadForArrayWithQuery.

/**
    * 
    * @throws Exception
    */
@Test
public void testLoadForArrayWithQuery() throws Exception {
    //create the table
    final String TABLE = "TABLE14";
    String ddl = "CREATE TABLE  " + TABLE + " ( ID INTEGER PRIMARY KEY, a_double_array double array[] , a_varchar_array varchar array, a_concat_str varchar, sep varchar)";
    conn.createStatement().execute(ddl);
    Double[] doubleArr = new Double[3];
    doubleArr[0] = 2.2;
    doubleArr[1] = 4.4;
    doubleArr[2] = 6.6;
    Array doubleArray = conn.createArrayOf("DOUBLE", doubleArr);
    Tuple doubleArrTuple = Storage.tuple(2.2d, 4.4d, 6.6d);
    Double[] doubleArr2 = new Double[2];
    doubleArr2[0] = 12.2;
    doubleArr2[1] = 22.2;
    Array doubleArray2 = conn.createArrayOf("DOUBLE", doubleArr2);
    Tuple doubleArrTuple2 = Storage.tuple(12.2d, 22.2d);
    String[] strArr = new String[4];
    strArr[0] = "ABC";
    strArr[1] = "DEF";
    strArr[2] = "GHI";
    strArr[3] = "JKL";
    Array strArray = conn.createArrayOf("VARCHAR", strArr);
    Tuple strArrTuple = Storage.tuple("ABC", "DEF", "GHI", "JKL");
    String[] strArr2 = new String[2];
    strArr2[0] = "ABC";
    strArr2[1] = "XYZ";
    Array strArray2 = conn.createArrayOf("VARCHAR", strArr2);
    Tuple strArrTuple2 = Storage.tuple("ABC", "XYZ");
    //upsert data.
    final String dml = "UPSERT INTO " + TABLE + " VALUES(?, ?, ?, ?, ?) ";
    PreparedStatement stmt = conn.prepareStatement(dml);
    stmt.setInt(1, 1);
    stmt.setArray(2, doubleArray);
    stmt.setArray(3, strArray);
    stmt.setString(4, "ONE,TWO,THREE");
    stmt.setString(5, ",");
    stmt.execute();
    stmt.setInt(1, 2);
    stmt.setArray(2, doubleArray2);
    stmt.setArray(3, strArray2);
    stmt.setString(4, "FOUR:five:six");
    stmt.setString(5, ":");
    stmt.execute();
    conn.commit();
    Tuple dynArrTuple = Storage.tuple("ONE", "TWO", "THREE");
    Tuple dynArrTuple2 = Storage.tuple("FOUR", "five", "six");
    //sql query
    final String sqlQuery = String.format(" SELECT ID, A_DOUBLE_ARRAY, A_VARCHAR_ARRAY, REGEXP_SPLIT(a_concat_str, sep) AS flattend_str FROM %s ", TABLE);
    final Data data = Storage.resetData(pigServer);
    List<Tuple> expectedList = new ArrayList<Tuple>();
    expectedList.add(Storage.tuple(1, 3L, 4L, dynArrTuple));
    expectedList.add(Storage.tuple(2, 2L, 2L, dynArrTuple2));
    final String load = String.format("A = load 'hbase://query/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", sqlQuery, zkQuorum);
    pigServer.setBatchOn();
    pigServer.registerQuery(load);
    pigServer.registerQuery("B = FOREACH A GENERATE ID, SIZE(A_DOUBLE_ARRAY), SIZE(A_VARCHAR_ARRAY), FLATTEND_STR;");
    pigServer.registerQuery("STORE B INTO 'out' using mock.Storage();");
    pigServer.executeBatch();
    List<Tuple> actualList = data.get("out");
    assertEquals(expectedList.size(), actualList.size());
    assertEquals(expectedList, actualList);
    Schema schema = pigServer.dumpSchema("A");
    List<FieldSchema> fields = schema.getFields();
    assertEquals(4, fields.size());
    assertTrue(fields.get(0).alias.equalsIgnoreCase("ID"));
    assertTrue(fields.get(0).type == DataType.INTEGER);
    assertTrue(fields.get(1).alias.equalsIgnoreCase("A_DOUBLE_ARRAY"));
    assertTrue(fields.get(1).type == DataType.TUPLE);
    assertTrue(fields.get(2).alias.equalsIgnoreCase("A_VARCHAR_ARRAY"));
    assertTrue(fields.get(2).type == DataType.TUPLE);
    assertTrue(fields.get(3).alias.equalsIgnoreCase("FLATTEND_STR"));
    assertTrue(fields.get(3).type == DataType.TUPLE);
    Iterator<Tuple> iterator = pigServer.openIterator("A");
    Tuple firstTuple = Storage.tuple(1, doubleArrTuple, strArrTuple, dynArrTuple);
    Tuple secondTuple = Storage.tuple(2, doubleArrTuple2, strArrTuple2, dynArrTuple2);
    List<Tuple> expectedRows = Lists.newArrayList(firstTuple, secondTuple);
    List<Tuple> actualRows = Lists.newArrayList();
    while (iterator.hasNext()) {
        Tuple tuple = iterator.next();
        actualRows.add(tuple);
    }
    assertEquals(expectedRows, actualRows);
}

Example 15 with FieldSchema

use of org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema in project hive by apache.

the class TestPigHBaseStorageHandler method testPigHBaseSchema.

@Test
public void testPigHBaseSchema() throws Exception {
    Initialize();
    String tableName = newTableName("MyTable");
    String databaseName = newTableName("MyDatabase");
    // Table name will be lower case unless specified by hbase.table.name property
    String hbaseTableName = "testTable";
    String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
    String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'";
    String deleteQuery = "DROP TABLE " + databaseName + "." + tableName;
    String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + "(key float, testqualifier1 string, testqualifier2 int) STORED BY " + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')" + " TBLPROPERTIES ('hbase.table.name'='" + hbaseTableName + "')";
    CommandProcessorResponse responseOne = driver.run(deleteQuery);
    assertEquals(0, responseOne.getResponseCode());
    CommandProcessorResponse responseTwo = driver.run(dbQuery);
    assertEquals(0, responseTwo.getResponseCode());
    CommandProcessorResponse responseThree = driver.run(tableQuery);
    Connection connection = null;
    Admin hAdmin = null;
    boolean doesTableExist = false;
    try {
        connection = ConnectionFactory.createConnection(getHbaseConf());
        hAdmin = connection.getAdmin();
        doesTableExist = hAdmin.tableExists(TableName.valueOf(hbaseTableName));
    } finally {
        if (hAdmin != null) {
            hAdmin.close();
        }
        if (connection != null) {
            connection.close();
        }
    }
    assertTrue(doesTableExist);
    PigServer server = new PigServer(ExecType.LOCAL, hcatConf.getAllProperties());
    server.registerQuery("A = load '" + databaseName + "." + tableName + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
    Schema dumpedASchema = server.dumpSchema("A");
    List<FieldSchema> fields = dumpedASchema.getFields();
    assertEquals(3, fields.size());
    assertEquals(DataType.FLOAT, fields.get(0).type);
    assertEquals("key", fields.get(0).alias.toLowerCase());
    assertEquals(DataType.CHARARRAY, fields.get(1).type);
    assertEquals("testQualifier1".toLowerCase(), fields.get(1).alias.toLowerCase());
    assertEquals(DataType.INTEGER, fields.get(2).type);
    assertEquals("testQualifier2".toLowerCase(), fields.get(2).alias.toLowerCase());
}