Examples with MessageType org.apache.parquet.schema.MessageType used on opensource projects

Example 26 with MessageType

use of org.apache.parquet.schema.MessageType in project hive by apache.

the class TestParquetFilterPredicate method testFilterColumnsThatDoNoExistOnSchemaHighOrder1.

@Test
public void testFilterColumnsThatDoNoExistOnSchemaHighOrder1() {
    MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; required int32 b; }");
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startOr().startAnd().equals("a", PredicateLeaf.Type.LONG, 1L).equals("none", PredicateLeaf.Type.LONG, 1L).end().startAnd().equals("a", PredicateLeaf.Type.LONG, 999L).equals("none", PredicateLeaf.Type.LONG, 999L).end().end().build();
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema);
    String expected = "or(eq(a, 1), eq(a, 999))";
    assertEquals(expected, p.toString());
}

Example 27 with MessageType

use of org.apache.parquet.schema.MessageType in project hive by apache.

the class TestConvertAstToSearchArg method testExpression2.

@Test
public void testExpression2() throws Exception {
    /* first_name is null or
       first_name <> 'sue' or
       id >= 12 or
       id <= 4; */
    String exprStr = "<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n" + "<java version=\"1.6.0_31\" class=\"java.beans.XMLDecoder\"> \n" + " <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "  <void property=\"children\"> \n" + "   <object class=\"java.util.ArrayList\"> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "              <void property=\"children\"> \n" + "               <object class=\"java.util.ArrayList\"> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                  <void property=\"column\"> \n" + "                   <string>first_name</string> \n" + "                  </void> \n" + "                  <void property=\"tabAlias\"> \n" + "                   <string>orc_people</string> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object id=\"PrimitiveTypeInfo0\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                    <void property=\"typeName\"> \n" + "                     <string>string</string> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "              <void property=\"genericUDF\"> \n" + "               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull\"/> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object id=\"PrimitiveTypeInfo1\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                <void property=\"typeName\"> \n" + "                 <string>boolean</string> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "              <void property=\"children\"> \n" + "               <object class=\"java.util.ArrayList\"> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                  <void property=\"column\"> \n" + "                   <string>first_name</string> \n" + "                  </void> \n" + "                  <void property=\"tabAlias\"> \n" + "                   <string>orc_people</string> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                  </void> \n" + "                  <void property=\"value\"> \n" + "                   <string>sue</string> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "              <void property=\"genericUDF\"> \n" + "               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual\"/> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo1\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>id</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object id=\"PrimitiveTypeInfo2\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                <void property=\"typeName\"> \n" + "                 <string>int</string> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <int>12</int> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "          <void property=\"column\"> \n" + "           <string>id</string> \n" + "          </void> \n" + "          <void property=\"tabAlias\"> \n" + "           <string>orc_people</string> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo2\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo2\"/> \n" + "          </void> \n" + "          <void property=\"value\"> \n" + "           <int>4</int> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "   </object> \n" + "  </void> \n" + "  <void property=\"genericUDF\"> \n" + "   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "  </void> \n" + "  <void property=\"typeInfo\"> \n" + "   <object idref=\"PrimitiveTypeInfo1\"/> \n" + "  </void> \n" + " </object> \n" + "</java> \n";
    SearchArgumentImpl sarg = (SearchArgumentImpl) ConvertAstToSearchArg.create(conf, getFuncDesc(exprStr));
    List<PredicateLeaf> leaves = sarg.getLeaves();
    assertEquals(4, leaves.size());
    String[] conditions = new String[] { "eq(first_name, null)", /* first_name is null  */
    "not(eq(first_name, Binary{\"sue\"}))", /* first_name <> 'sue' */
    "not(lt(id, 12))", /* id >= 12            */
    "lteq(id, 4)" /* id <= 4             */
    };
    MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema);
    String expected = String.format("or(or(or(%1$s, %2$s), %3$s), %4$s)", conditions);
    assertEquals(expected, p.toString());
    PredicateLeaf leaf = leaves.get(0);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.IS_NULL, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals(null, leaf.getLiteral());
    assertEquals(null, leaf.getLiteralList());
    leaf = leaves.get(1);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("sue", leaf.getLiteral());
    leaf = leaves.get(2);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(12L, leaf.getLiteral());
    leaf = leaves.get(3);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(4L, leaf.getLiteral());
    assertEquals("(or leaf-0 (not leaf-1) (not leaf-2) leaf-3)", sarg.getExpression().toString());
    assertNoSharedNodes(sarg.getExpression(), Sets.<ExpressionTree>newIdentityHashSet());
    assertEquals(TruthValue.NO, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES, TruthValue.NO)));
    assertEquals(TruthValue.YES, sarg.evaluate(values(TruthValue.YES, TruthValue.YES, TruthValue.YES, TruthValue.NO)));
    assertEquals(TruthValue.YES, sarg.evaluate(values(TruthValue.NO, TruthValue.NO, TruthValue.YES, TruthValue.NO)));
    assertEquals(TruthValue.YES, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.NO, TruthValue.NO)));
    assertEquals(TruthValue.YES, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES, TruthValue.YES)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NULL, TruthValue.YES, TruthValue.YES, TruthValue.NO)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.NULL, TruthValue.YES, TruthValue.NO)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.NULL, TruthValue.NO)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES, TruthValue.NULL)));
    assertEquals(TruthValue.YES_NO, sarg.evaluate(values(TruthValue.NO, TruthValue.YES_NO, TruthValue.YES, TruthValue.YES_NO)));
    assertEquals(TruthValue.NO_NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.YES_NULL, TruthValue.YES, TruthValue.NO_NULL)));
    assertEquals(TruthValue.YES_NULL, sarg.evaluate(values(TruthValue.YES_NULL, TruthValue.YES_NO_NULL, TruthValue.YES, TruthValue.NULL)));
    assertEquals(TruthValue.YES_NO_NULL, sarg.evaluate(values(TruthValue.NO_NULL, TruthValue.YES_NO_NULL, TruthValue.YES, TruthValue.NO)));
}

Example 28 with MessageType

use of org.apache.parquet.schema.MessageType in project hive by apache.

the class TestConvertAstToSearchArg method testExpression4.

@Test
public void testExpression4() throws Exception {
    /* id <> 12 and
       first_name in ('john', 'sue') and
       id in (34,50) */
    String exprStr = "<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n" + "<java version=\"1.6.0_31\" class=\"java.beans.XMLDecoder\"> \n" + " <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "  <void property=\"children\"> \n" + "   <object class=\"java.util.ArrayList\"> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>id</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object id=\"PrimitiveTypeInfo0\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                <void property=\"typeName\"> \n" + "                 <string>int</string> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <int>12</int> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object id=\"PrimitiveTypeInfo1\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "            <void property=\"typeName\"> \n" + "             <string>boolean</string> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>first_name</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object id=\"PrimitiveTypeInfo2\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                <void property=\"typeName\"> \n" + "                 <string>string</string> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <string>john</string> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <string>sue</string> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "          <void property=\"column\"> \n" + "           <string>id</string> \n" + "          </void> \n" + "          <void property=\"tabAlias\"> \n" + "           <string>orc_people</string> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo0\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo0\"/> \n" + "          </void> \n" + "          <void property=\"value\"> \n" + "           <int>34</int> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo0\"/> \n" + "          </void> \n" + "          <void property=\"value\"> \n" + "           <int>50</int> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "   </object> \n" + "  </void> \n" + "  <void property=\"genericUDF\"> \n" + "   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "  </void> \n" + "  <void property=\"typeInfo\"> \n" + "   <object idref=\"PrimitiveTypeInfo1\"/> \n" + "  </void> \n" + " </object> \n" + "</java> \n" + "\n";
    SearchArgumentImpl sarg = (SearchArgumentImpl) ConvertAstToSearchArg.create(conf, getFuncDesc(exprStr));
    List<PredicateLeaf> leaves = sarg.getLeaves();
    assertEquals(3, leaves.size());
    String[] conditions = new String[] { "not(eq(id, 12))", /* id <> 12 */
    "or(eq(first_name, Binary{\"john\"}), eq(first_name, Binary{\"sue\"}))", /* first_name in
      ('john', 'sue') */
    "or(eq(id, 34), eq(id, 50))" /* id in (34,50) */
    };
    MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema);
    String expected = String.format("and(and(%1$s, %2$s), %3$s)", conditions);
    assertEquals(expected, p.toString());
    PredicateLeaf leaf = leaves.get(0);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(12L, leaf.getLiteral());
    leaf = leaves.get(1);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.IN, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("john", leaf.getLiteralList().get(0));
    assertEquals("sue", leaf.getLiteralList().get(1));
    leaf = leaves.get(2);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.IN, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(34L, leaf.getLiteralList().get(0));
    assertEquals(50L, leaf.getLiteralList().get(1));
    assertEquals("(and (not leaf-0) leaf-1 leaf-2)", sarg.getExpression().toString());
    assertNoSharedNodes(sarg.getExpression(), Sets.<ExpressionTree>newIdentityHashSet());
    assertEquals(TruthValue.YES, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NULL, TruthValue.YES, TruthValue.YES)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.NULL, TruthValue.YES)));
    assertEquals(TruthValue.NO, sarg.evaluate(values(TruthValue.YES, TruthValue.YES, TruthValue.YES)));
    assertEquals(TruthValue.NO, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.NO)));
    assertEquals(TruthValue.NO, sarg.evaluate(values(TruthValue.NO, TruthValue.YES_NULL, TruthValue.NO)));
    assertEquals(TruthValue.NO_NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.NULL, TruthValue.YES_NO_NULL)));
    assertEquals(TruthValue.NO_NULL, sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.NO_NULL)));
}

Example 29 with MessageType

use of org.apache.parquet.schema.MessageType in project hive by apache.

the class TestConvertAstToSearchArg method testExpression3.

@Test
public void testExpression3() throws Exception {
    /* (id between 23 and 45) and
       first_name = 'alan' and
       substr('xxxxx', 3) == first_name and
       'smith' = last_name and
       substr(first_name, 3) == 'yyy' */
    String exprStr = "<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n" + "<java version=\"1.6.0_31\" class=\"java.beans.XMLDecoder\"> \n" + " <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "  <void property=\"children\"> \n" + "   <object class=\"java.util.ArrayList\"> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "              <void property=\"children\"> \n" + "               <object class=\"java.util.ArrayList\"> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                  <void property=\"children\"> \n" + "                   <object class=\"java.util.ArrayList\"> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object id=\"PrimitiveTypeInfo0\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                        <void property=\"typeName\"> \n" + "                         <string>boolean</string> \n" + "                        </void> \n" + "                       </object> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <boolean>false</boolean> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                      <void property=\"column\"> \n" + "                       <string>id</string> \n" + "                      </void> \n" + "                      <void property=\"tabAlias\"> \n" + "                       <string>orc_people</string> \n" + "                      </void> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object id=\"PrimitiveTypeInfo1\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                        <void property=\"typeName\"> \n" + "                         <string>int</string> \n" + "                        </void> \n" + "                       </object> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <int>23</int> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <int>45</int> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                  <void property=\"genericUDF\"> \n" + "                   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween\"/> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                  <void property=\"children\"> \n" + "                   <object class=\"java.util.ArrayList\"> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                      <void property=\"column\"> \n" + "                       <string>first_name</string> \n" + "                      </void> \n" + "                      <void property=\"tabAlias\"> \n" + "                       <string>orc_people</string> \n" + "                      </void> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object id=\"PrimitiveTypeInfo2\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                        <void property=\"typeName\"> \n" + "                         <string>string</string> \n" + "                        </void> \n" + "                       </object> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <string>alan</string> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                  <void property=\"genericUDF\"> \n" + "                   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual\"/> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "              <void property=\"genericUDF\"> \n" + "               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "              <void property=\"children\"> \n" + "               <object class=\"java.util.ArrayList\"> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                  <void property=\"children\"> \n" + "                   <object class=\"java.util.ArrayList\"> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <string>xxxxx</string> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <int>3</int> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                  <void property=\"genericUDF\"> \n" + "                   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge\"> \n" + "                    <void property=\"udfClassName\"> \n" + "                     <string>org.apache.hadoop.hive.ql.udf.UDFSubstr</string> \n" + "                    </void> \n" + "                    <void property=\"udfName\"> \n" + "                     <string>substr</string> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                  <void property=\"column\"> \n" + "                   <string>first_name</string> \n" + "                  </void> \n" + "                  <void property=\"tabAlias\"> \n" + "                   <string>orc_people</string> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "              <void property=\"genericUDF\"> \n" + "               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual\"/> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo0\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <string>smith</string> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>last_name</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo0\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo0\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>first_name</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo1\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <int>3</int> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge\"> \n" + "            <void property=\"udfClassName\"> \n" + "             <string>org.apache.hadoop.hive.ql.udf.UDFSubstr</string> \n" + "            </void> \n" + "            <void property=\"udfName\"> \n" + "             <string>substr</string> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo2\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo2\"/> \n" + "          </void> \n" + "          <void property=\"value\"> \n" + "           <string>yyy</string> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo0\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "   </object> \n" + "  </void> \n" + "  <void property=\"genericUDF\"> \n" + "   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "  </void> \n" + "  <void property=\"typeInfo\"> \n" + "   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "  </void> \n" + " </object> \n" + "</java> \n";
    SearchArgumentImpl sarg = (SearchArgumentImpl) ConvertAstToSearchArg.create(conf, getFuncDesc(exprStr));
    List<PredicateLeaf> leaves = sarg.getLeaves();
    assertEquals(3, leaves.size());
    String[] conditions = new String[] { "lteq(id, 45)", /* id between 23 and 45 */
    "not(lt(id, 23))", /* id between 23 and 45 */
    "eq(first_name, Binary{\"alan\"})", /* first_name = 'alan'  */
    "eq(last_name, Binary{\"smith\"})" /* 'smith' = last_name  */
    };
    MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; required binary last_name;}");
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema);
    String expected = String.format("and(and(and(%1$s, %2$s), %3$s), %4$s)", conditions);
    assertEquals(expected, p.toString());
    PredicateLeaf leaf = leaves.get(0);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.BETWEEN, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(null, leaf.getLiteral());
    assertEquals(23L, leaf.getLiteralList().get(0));
    assertEquals(45L, leaf.getLiteralList().get(1));
    leaf = leaves.get(1);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("alan", leaf.getLiteral());
    leaf = leaves.get(2);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator());
    assertEquals("last_name", leaf.getColumnName());
    assertEquals("smith", leaf.getLiteral());
    assertEquals("(and leaf-0 leaf-1 leaf-2)", sarg.getExpression().toString());
    assertNoSharedNodes(sarg.getExpression(), Sets.<ExpressionTree>newIdentityHashSet());
}

Example 30 with MessageType

use of org.apache.parquet.schema.MessageType in project hive by apache.

the class TestDataWritableWriter method writeParquetRecord.

private void writeParquetRecord(String schema, ParquetHiveRecord record) throws SerDeException {
    MessageType fileSchema = MessageTypeParser.parseMessageType(schema);
    DataWritableWriter hiveParquetWriter = new DataWritableWriter(mockRecordConsumer, fileSchema);
    hiveParquetWriter.write(record);
}