Examples with And org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And used on opensource projects

Example 46 with And

use of org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And in project hive by apache.

the class TestParquetFilterPredicate method testFilterStringColumnWithWhiteSpaces.

@Test
public void testFilterStringColumnWithWhiteSpaces() throws Exception {
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startAnd().lessThan("a", PredicateLeaf.Type.STRING, new HiveVarchar(" apple  ", 10).toString()).lessThanEquals("b", PredicateLeaf.Type.STRING, new HiveVarchar(" pear", 10).toString()).equals("c", PredicateLeaf.Type.STRING, new HiveVarchar("orange ", 10).toString()).end().build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required binary a; required binary b;" + " required binary c;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("a", TypeInfoFactory.getPrimitiveTypeInfo("string"));
    columnTypes.put("b", TypeInfoFactory.getPrimitiveTypeInfo("string"));
    columnTypes.put("c", TypeInfoFactory.getPrimitiveTypeInfo("string"));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String expected = "and(and(" + "lt(a, Binary{\" apple  \"}), " + "lteq(b, Binary{\" pear\"})), " + "eq(c, Binary{\"orange \"}))";
    assertEquals(expected, p.toString());
}

Example 47 with And

use of org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And in project hive by apache.

the class TestParquetFilterPredicate method testFilterMoreComplexVarCharColumn.

@Test
public void testFilterMoreComplexVarCharColumn() throws Exception {
    // ((a=pear or a<=cherry) and (b=orange)) and (c=banana or d<cherry)
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startAnd().startAnd().startOr().equals("a", PredicateLeaf.Type.STRING, new HiveVarchar("pear", 10).toString()).lessThanEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("cherry", 10).toString()).end().equals("b", PredicateLeaf.Type.STRING, new HiveVarchar("orange", 10).toString()).end().startOr().equals("c", PredicateLeaf.Type.STRING, new HiveVarchar("banana", 10).toString()).lessThan("d", PredicateLeaf.Type.STRING, new HiveVarchar("cherry", 10).toString()).end().end().build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required binary a; required binary b;" + " required binary c; required binary d;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("a", TypeInfoFactory.getVarcharTypeInfo(10));
    columnTypes.put("b", TypeInfoFactory.getVarcharTypeInfo(10));
    columnTypes.put("c", TypeInfoFactory.getVarcharTypeInfo(10));
    columnTypes.put("d", TypeInfoFactory.getVarcharTypeInfo(10));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String expected = "and(and(or(eq(a, Binary{\"pear\"}), lteq(a, Binary{\"cherry\"})), " + "eq(b, Binary{\"orange\"})), " + "or(eq(c, Binary{\"banana\"}), lt(d, Binary{\"cherry\"})))";
    assertEquals(expected, p.toString());
}

Example 48 with And

use of org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And in project hive by apache.

the class TestParquetFilterPredicate method testFilterCharColumnWhiteSpacePostfix.

@Test
public void testFilterCharColumnWhiteSpacePostfix() throws Exception {
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startAnd().lessThan("a", PredicateLeaf.Type.STRING, new HiveChar("apple  ", 10).toString()).lessThanEquals("b", PredicateLeaf.Type.STRING, new HiveChar("pear  ", 10).toString()).equals("c", PredicateLeaf.Type.STRING, new HiveChar("orange  ", 10).toString()).nullSafeEquals("d", PredicateLeaf.Type.STRING, new HiveChar("pineapple ", 10).toString()).end().build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required binary a; required binary b;" + " required binary c; required binary d;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("a", TypeInfoFactory.getCharTypeInfo(10));
    columnTypes.put("b", TypeInfoFactory.getCharTypeInfo(10));
    columnTypes.put("c", TypeInfoFactory.getCharTypeInfo(10));
    columnTypes.put("d", TypeInfoFactory.getCharTypeInfo(10));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String expected = "and(and(and(" + "lt(a, Binary{\"apple\"}), " + "lteq(b, Binary{\"pear\"})), " + "eq(c, Binary{\"orange\"})), " + "eq(d, Binary{\"pineapple\"}))";
    assertEquals(expected, p.toString());
}

Example 49 with And

use of org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And in project hive by apache.

the class TestConvertAstToSearchArg method testExpression1.

@Test
public void testExpression1() throws Exception {
    // first_name = 'john' or
    // 'greg' < first_name or
    // 'alan' > first_name or
    // id > 12 or
    // 13 < id or
    // id < 15 or
    // 16 > id or
    // (id <=> 30 and first_name <=> 'owen')
    String exprStr = "<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n" + "<java version=\"1.6.0_31\" class=\"java.beans.XMLDecoder\"> \n" + " <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "  <void property=\"children\"> \n" + "   <object class=\"java.util.ArrayList\"> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "              <void property=\"children\"> \n" + "               <object class=\"java.util.ArrayList\"> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                  <void property=\"children\"> \n" + "                   <object class=\"java.util.ArrayList\"> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                      <void property=\"children\"> \n" + "                       <object class=\"java.util.ArrayList\"> \n" + "                        <void method=\"add\"> \n" + "                         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                          <void property=\"children\"> \n" + "                           <object class=\"java.util.ArrayList\"> \n" + "                            <void method=\"add\"> \n" + "                             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                              <void property=\"children\"> \n" + "                               <object class=\"java.util.ArrayList\"> \n" + "                                <void method=\"add\"> \n" + "                                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                                  <void property=\"column\"> \n" + "                                   <string>first_name</string> \n" + "                                  </void> \n" + "                                  <void property=\"tabAlias\"> \n" + "                                   <string>orc_people</string> \n" + "                                  </void> \n" + "                                  <void property=\"typeInfo\"> \n" + "                                   <object id=\"PrimitiveTypeInfo0\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                                    <void property=\"typeName\"> \n" + "                                     <string>string</string> \n" + "                                    </void> \n" + "                                   </object> \n" + "                                  </void> \n" + "                                 </object> \n" + "                                </void> \n" + "                                <void method=\"add\"> \n" + "                                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                                  <void property=\"typeInfo\"> \n" + "                                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                                  </void> \n" + "                                  <void property=\"value\"> \n" + "                                   <string>john</string> \n" + "                                  </void> \n" + "                                 </object> \n" + "                                </void> \n" + "                               </object> \n" + "                              </void> \n" + "                              <void property=\"genericUDF\"> \n" + "                               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual\"/> \n" + "                              </void> \n" + "                              <void property=\"typeInfo\"> \n" + "                               <object id=\"PrimitiveTypeInfo1\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                                <void property=\"typeName\"> \n" + "                                 <string>boolean</string> \n" + "                                </void> \n" + "                               </object> \n" + "                              </void> \n" + "                             </object> \n" + "                            </void> \n" + "                            <void method=\"add\"> \n" + "                             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                              <void property=\"children\"> \n" + "                               <object class=\"java.util.ArrayList\"> \n" + "                                <void method=\"add\"> \n" + "                                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                                  <void property=\"typeInfo\"> \n" + "                                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                                  </void> \n" + "                                  <void property=\"value\"> \n" + "                                   <string>greg</string> \n" + "                                  </void> \n" + "                                 </object> \n" + "                                </void> \n" + "                                <void method=\"add\"> \n" + "                                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                                  <void property=\"column\"> \n" + "                                   <string>first_name</string> \n" + "                                  </void> \n" + "                                  <void property=\"tabAlias\"> \n" + "                                   <string>orc_people</string> \n" + "                                  </void> \n" + "                                  <void property=\"typeInfo\"> \n" + "                                   <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                                  </void> \n" + "                                 </object> \n" + "                                </void> \n" + "                               </object> \n" + "                              </void> \n" + "                              <void property=\"genericUDF\"> \n" + "                               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan\"/> \n" + "                              </void> \n" + "                              <void property=\"typeInfo\"> \n" + "                               <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                              </void> \n" + "                             </object> \n" + "                            </void> \n" + "                           </object> \n" + "                          </void> \n" + "                          <void property=\"genericUDF\"> \n" + "                           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "                          </void> \n" + "                          <void property=\"typeInfo\"> \n" + "                           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                          </void> \n" + "                         </object> \n" + "                        </void> \n" + "                        <void method=\"add\"> \n" + "                         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                          <void property=\"children\"> \n" + "                           <object class=\"java.util.ArrayList\"> \n" + "                            <void method=\"add\"> \n" + "                             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                              <void property=\"typeInfo\"> \n" + "                               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                              </void> \n" + "                              <void property=\"value\"> \n" + "                               <string>alan</string> \n" + "                              </void> \n" + "                             </object> \n" + "                            </void> \n" + "                            <void method=\"add\"> \n" + "                             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                              <void property=\"column\"> \n" + "                               <string>first_name</string> \n" + "                              </void> \n" + "                              <void property=\"tabAlias\"> \n" + "                               <string>orc_people</string> \n" + "                              </void> \n" + "                              <void property=\"typeInfo\"> \n" + "                               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "                              </void> \n" + "                             </object> \n" + "                            </void> \n" + "                           </object> \n" + "                          </void> \n" + "                          <void property=\"genericUDF\"> \n" + "                           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan\"/> \n" + "                          </void> \n" + "                          <void property=\"typeInfo\"> \n" + "                           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                          </void> \n" + "                         </object> \n" + "                        </void> \n" + "                       </object> \n" + "                      </void> \n" + "                      <void property=\"genericUDF\"> \n" + "                       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "                      </void> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                      <void property=\"children\"> \n" + "                       <object class=\"java.util.ArrayList\"> \n" + "                        <void method=\"add\"> \n" + "                         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                          <void property=\"column\"> \n" + "                           <string>id</string> \n" + "                          </void> \n" + "                          <void property=\"tabAlias\"> \n" + "                           <string>orc_people</string> \n" + "                          </void> \n" + "                          <void property=\"typeInfo\"> \n" + "                           <object id=\"PrimitiveTypeInfo2\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "                            <void property=\"typeName\"> \n" + "                             <string>int</string> \n" + "                            </void> \n" + "                           </object> \n" + "                          </void> \n" + "                         </object> \n" + "                        </void> \n" + "                        <void method=\"add\"> \n" + "                         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                          <void property=\"typeInfo\"> \n" + "                           <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                          </void> \n" + "                          <void property=\"value\"> \n" + "                           <int>12</int> \n" + "                          </void> \n" + "                         </object> \n" + "                        </void> \n" + "                       </object> \n" + "                      </void> \n" + "                      <void property=\"genericUDF\"> \n" + "                       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan\"/> \n" + "                      </void> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                  <void property=\"genericUDF\"> \n" + "                   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "                  <void property=\"children\"> \n" + "                   <object class=\"java.util.ArrayList\"> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                      </void> \n" + "                      <void property=\"value\"> \n" + "                       <int>13</int> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                    <void method=\"add\"> \n" + "                     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                      <void property=\"column\"> \n" + "                       <string>id</string> \n" + "                      </void> \n" + "                      <void property=\"tabAlias\"> \n" + "                       <string>orc_people</string> \n" + "                      </void> \n" + "                      <void property=\"typeInfo\"> \n" + "                       <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                      </void> \n" + "                     </object> \n" + "                    </void> \n" + "                   </object> \n" + "                  </void> \n" + "                  <void property=\"genericUDF\"> \n" + "                   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan\"/> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo1\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "              <void property=\"genericUDF\"> \n" + "               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo1\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "              <void property=\"children\"> \n" + "               <object class=\"java.util.ArrayList\"> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "                  <void property=\"column\"> \n" + "                   <string>id</string> \n" + "                  </void> \n" + "                  <void property=\"tabAlias\"> \n" + "                   <string>orc_people</string> \n" + "                  </void> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "                <void method=\"add\"> \n" + "                 <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "                  <void property=\"typeInfo\"> \n" + "                   <object idref=\"PrimitiveTypeInfo2\"/> \n" + "                  </void> \n" + "                  <void property=\"value\"> \n" + "                   <int>15</int> \n" + "                  </void> \n" + "                 </object> \n" + "                </void> \n" + "               </object> \n" + "              </void> \n" + "              <void property=\"genericUDF\"> \n" + "               <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan\"/> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo1\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <int>16</int> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>id</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>id</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo2\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <int>30</int> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>first_name</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <string>owen</string> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "   </object> \n" + "  </void> \n" + "  <void property=\"genericUDF\"> \n" + "   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr\"/> \n" + "  </void> \n" + "  <void property=\"typeInfo\"> \n" + "   <object idref=\"PrimitiveTypeInfo1\"/> \n" + "  </void> \n" + " </object> \n" + "</java> \n";
    SearchArgumentImpl sarg = (SearchArgumentImpl) ConvertAstToSearchArg.create(conf, getFuncDesc(exprStr));
    List<PredicateLeaf> leaves = sarg.getLeaves();
    assertEquals(9, leaves.size());
    MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("id", TypeInfoFactory.getPrimitiveTypeInfo("int"));
    columnTypes.put("first_name", TypeInfoFactory.getPrimitiveTypeInfo("string"));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String[] conditions = new String[] { "eq(first_name, Binary{\"john\"})", /* first_name = 'john' */
    "not(lteq(first_name, Binary{\"greg\"}))", /* 'greg' < first_name */
    "lt(first_name, Binary{\"alan\"})", /* 'alan' > first_name */
    "not(lteq(id, 12))", /* id > 12 or */
    "not(lteq(id, 13))", /* 13 < id or */
    "lt(id, 15)", /* id < 15 or */
    "lt(id, 16)", /* 16 > id or */
    "eq(id, 30)", /* id <=> 30 */
    "eq(first_name, Binary{\"owen\"})" /* first_name <=> 'owen' */
    };
    String expected = String.format("and(or(or(or(or(or(or(or(%1$s, %2$s), %3$s), %4$s), %5$s), %6$s), %7$s), %8$s), " + "or(or(or(or(or(or(or(%1$s, %2$s), %3$s), %4$s), %5$s), %6$s), %7$s), %9$s))", conditions);
    assertEquals(expected, p.toString());
    PredicateLeaf leaf = leaves.get(0);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("john", leaf.getLiteral());
    leaf = leaves.get(1);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("greg", leaf.getLiteral());
    leaf = leaves.get(2);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("alan", leaf.getLiteral());
    leaf = leaves.get(3);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(12L, leaf.getLiteral());
    leaf = leaves.get(4);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(13L, leaf.getLiteral());
    leaf = leaves.get(5);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(15L, leaf.getLiteral());
    leaf = leaves.get(6);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(16L, leaf.getLiteral());
    leaf = leaves.get(7);
    assertEquals(PredicateLeaf.Type.LONG, leaf.getType());
    assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS, leaf.getOperator());
    assertEquals("id", leaf.getColumnName());
    assertEquals(30L, leaf.getLiteral());
    leaf = leaves.get(8);
    assertEquals(PredicateLeaf.Type.STRING, leaf.getType());
    assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS, leaf.getOperator());
    assertEquals("first_name", leaf.getColumnName());
    assertEquals("owen", leaf.getLiteral());
    assertEquals("(and (or leaf-0 (not leaf-1) leaf-2 (not leaf-3)" + " (not leaf-4) leaf-5 leaf-6 leaf-7)" + " (or leaf-0 (not leaf-1) leaf-2 (not leaf-3)" + " (not leaf-4) leaf-5 leaf-6 leaf-8))", sarg.getExpression().toOldString());
    assertNoSharedNodes(sarg.getExpression(), Sets.<ExpressionTree>newIdentityHashSet());
}

Example 50 with And

use of org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And in project hive by apache.

the class TestConvertAstToSearchArg method testExpression10.

@Test
public void testExpression10() throws Exception {
    /* id >= 10 and not (10 > id) */
    String exprStr = "<?xml version=\"1.0\" encoding=\"UTF-8\"?> \n" + "<java version=\"1.6.0_31\" class=\"java.beans.XMLDecoder\"> \n" + " <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "  <void property=\"children\"> \n" + "   <object class=\"java.util.ArrayList\"> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "          <void property=\"column\"> \n" + "           <string>id</string> \n" + "          </void> \n" + "          <void property=\"tabAlias\"> \n" + "           <string>orc_people</string> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object id=\"PrimitiveTypeInfo0\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "            <void property=\"typeName\"> \n" + "             <string>int</string> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo0\"/> \n" + "          </void> \n" + "          <void property=\"value\"> \n" + "           <int>10</int> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object id=\"PrimitiveTypeInfo1\" class=\"org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo\"> \n" + "        <void property=\"typeName\"> \n" + "         <string>boolean</string> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "    <void method=\"add\"> \n" + "     <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "      <void property=\"children\"> \n" + "       <object class=\"java.util.ArrayList\"> \n" + "        <void method=\"add\"> \n" + "         <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc\"> \n" + "          <void property=\"children\"> \n" + "           <object class=\"java.util.ArrayList\"> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc\"> \n" + "              <void property=\"column\"> \n" + "               <string>id</string> \n" + "              </void> \n" + "              <void property=\"tabAlias\"> \n" + "               <string>orc_people</string> \n" + "              </void> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "            <void method=\"add\"> \n" + "             <object class=\"org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc\"> \n" + "              <void property=\"typeInfo\"> \n" + "               <object idref=\"PrimitiveTypeInfo0\"/> \n" + "              </void> \n" + "              <void property=\"value\"> \n" + "               <int>10</int> \n" + "              </void> \n" + "             </object> \n" + "            </void> \n" + "           </object> \n" + "          </void> \n" + "          <void property=\"genericUDF\"> \n" + "           <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan\"/> \n" + "          </void> \n" + "          <void property=\"typeInfo\"> \n" + "           <object idref=\"PrimitiveTypeInfo1\"/> \n" + "          </void> \n" + "         </object> \n" + "        </void> \n" + "       </object> \n" + "      </void> \n" + "      <void property=\"genericUDF\"> \n" + "       <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot\"/> \n" + "      </void> \n" + "      <void property=\"typeInfo\"> \n" + "       <object idref=\"PrimitiveTypeInfo1\"/> \n" + "      </void> \n" + "     </object> \n" + "    </void> \n" + "   </object> \n" + "  </void> \n" + "  <void property=\"genericUDF\"> \n" + "   <object class=\"org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd\"/> \n" + "  </void> \n" + "  <void property=\"typeInfo\"> \n" + "   <object idref=\"PrimitiveTypeInfo1\"/> \n" + "  </void> \n" + " </object> \n" + "</java>";
    SearchArgumentImpl sarg = (SearchArgumentImpl) ConvertAstToSearchArg.create(conf, getFuncDesc(exprStr));
    List<PredicateLeaf> leaves = sarg.getLeaves();
    assertEquals(1, leaves.size());
    MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 id;" + " required binary first_name; }");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("id", TypeInfoFactory.getPrimitiveTypeInfo("int"));
    columnTypes.put("first_name", TypeInfoFactory.getPrimitiveTypeInfo("string"));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String expected = "and(not(lt(id, 10)), not(lt(id, 10)))";
    assertEquals(expected, p.toString());
    assertEquals(PredicateLeaf.Type.LONG, leaves.get(0).getType());
    assertEquals(PredicateLeaf.Operator.LESS_THAN, leaves.get(0).getOperator());
    assertEquals("id", leaves.get(0).getColumnName());
    assertEquals(10L, leaves.get(0).getLiteral());
    assertEquals("(and (not leaf-0) (not leaf-0))", sarg.getExpression().toOldString());
    assertNoSharedNodes(sarg.getExpression(), Sets.<ExpressionTree>newIdentityHashSet());
    assertEquals(TruthValue.NO, sarg.evaluate(values(TruthValue.YES)));
    assertEquals(TruthValue.YES, sarg.evaluate(values(TruthValue.NO)));
    assertEquals(TruthValue.NULL, sarg.evaluate(values(TruthValue.NULL)));
    assertEquals(TruthValue.NO_NULL, sarg.evaluate(values(TruthValue.YES_NULL)));
    assertEquals(TruthValue.YES_NULL, sarg.evaluate(values(TruthValue.NO_NULL)));
    assertEquals(TruthValue.YES_NO, sarg.evaluate(values(TruthValue.YES_NO)));
    assertEquals(TruthValue.YES_NO_NULL, sarg.evaluate(values(TruthValue.YES_NO_NULL)));
}