use of org.apache.hadoop.io.FloatWritable in project nutch by apache.
the class CrawlDatum method evaluate.
public boolean evaluate(Expression expr, String url) {
if (expr != null && url != null) {
// Create a context and add data
JexlContext jcontext = new MapContext();
// https://issues.apache.org/jira/browse/NUTCH-2229
jcontext.set("url", url);
jcontext.set("status", getStatusName(getStatus()));
jcontext.set("fetchTime", (long) (getFetchTime()));
jcontext.set("modifiedTime", (long) (getModifiedTime()));
jcontext.set("retries", getRetriesSinceFetch());
jcontext.set("interval", new Integer(getFetchInterval()));
jcontext.set("score", getScore());
jcontext.set("signature", StringUtil.toHexString(getSignature()));
// Set metadata variables
for (Map.Entry<Writable, Writable> entry : getMetaData().entrySet()) {
Object value = entry.getValue();
Text tkey = (Text) entry.getKey();
if (value instanceof FloatWritable) {
FloatWritable fvalue = (FloatWritable) value;
jcontext.set(tkey.toString(), fvalue.get());
}
if (value instanceof IntWritable) {
IntWritable ivalue = (IntWritable) value;
jcontext.set(tkey.toString(), ivalue.get());
}
if (value instanceof Text) {
Text tvalue = (Text) value;
jcontext.set(tkey.toString().replace("-", "_"), tvalue.toString());
}
if (value instanceof ProtocolStatus) {
ProtocolStatus pvalue = (ProtocolStatus) value;
jcontext.set(tkey.toString().replace("-", "_"), pvalue.toString());
}
}
try {
if (Boolean.TRUE.equals(expr.evaluate(jcontext))) {
return true;
}
} catch (Exception e) {
//
}
}
return false;
}
use of org.apache.hadoop.io.FloatWritable in project hive by apache.
the class TestObjectInspectorConverters method testObjectInspectorConverters.
public void testObjectInspectorConverters() throws Throwable {
try {
// Boolean
Converter booleanConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
assertEquals("BooleanConverter", new BooleanWritable(false), booleanConverter.convert(Integer.valueOf(0)));
assertEquals("BooleanConverter", new BooleanWritable(true), booleanConverter.convert(Integer.valueOf(1)));
assertEquals("BooleanConverter", null, booleanConverter.convert(null));
// Byte
Converter byteConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableByteObjectInspector);
assertEquals("ByteConverter", new ByteWritable((byte) 0), byteConverter.convert(Integer.valueOf(0)));
assertEquals("ByteConverter", new ByteWritable((byte) 1), byteConverter.convert(Integer.valueOf(1)));
assertEquals("ByteConverter", null, byteConverter.convert(null));
// Short
Converter shortConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableShortObjectInspector);
assertEquals("ShortConverter", new ShortWritable((short) 0), shortConverter.convert(Integer.valueOf(0)));
assertEquals("ShortConverter", new ShortWritable((short) 1), shortConverter.convert(Integer.valueOf(1)));
assertEquals("ShortConverter", null, shortConverter.convert(null));
// Int
Converter intConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableIntObjectInspector);
assertEquals("IntConverter", new IntWritable(0), intConverter.convert(Integer.valueOf(0)));
assertEquals("IntConverter", new IntWritable(1), intConverter.convert(Integer.valueOf(1)));
assertEquals("IntConverter", null, intConverter.convert(null));
// Long
Converter longConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector);
assertEquals("LongConverter", new LongWritable(0), longConverter.convert(Integer.valueOf(0)));
assertEquals("LongConverter", new LongWritable(1), longConverter.convert(Integer.valueOf(1)));
assertEquals("LongConverter", null, longConverter.convert(null));
// Float
Converter floatConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
assertEquals("LongConverter", new FloatWritable(0), floatConverter.convert(Integer.valueOf(0)));
assertEquals("LongConverter", new FloatWritable(1), floatConverter.convert(Integer.valueOf(1)));
assertEquals("LongConverter", null, floatConverter.convert(null));
// Double
Converter doubleConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
assertEquals("DoubleConverter", new DoubleWritable(0), doubleConverter.convert(Integer.valueOf(0)));
assertEquals("DoubleConverter", new DoubleWritable(1), doubleConverter.convert(Integer.valueOf(1)));
assertEquals("DoubleConverter", null, doubleConverter.convert(null));
// Char
Converter charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
assertEquals("CharConverter", new HiveChar("TRUE", -1), charConverter.convert(Boolean.valueOf(true)));
assertEquals("CharConverter", new HiveChar("FALSE", -1), charConverter.convert(Boolean.valueOf(false)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("TRUE", -1)), charConverter.convert(Boolean.valueOf(true)));
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("FALSE", -1)), charConverter.convert(Boolean.valueOf(false)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
assertEquals("CharConverter", new HiveChar("0", -1), charConverter.convert(Integer.valueOf(0)));
assertEquals("CharConverter", new HiveChar("1", -1), charConverter.convert(Integer.valueOf(1)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("0", -1)), charConverter.convert(Integer.valueOf(0)));
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("1", -1)), charConverter.convert(Integer.valueOf(1)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
assertEquals("CharConverter", new HiveChar("hive", -1), charConverter.convert(String.valueOf("hive")));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("hive", -1)), charConverter.convert(String.valueOf("hive")));
// VarChar
Converter varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarchar("TRUE", -1), varcharConverter.convert(Boolean.valueOf(true)));
assertEquals("VarCharConverter", new HiveVarchar("FALSE", -1), varcharConverter.convert(Boolean.valueOf(false)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("TRUE", -1)), varcharConverter.convert(Boolean.valueOf(true)));
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("FALSE", -1)), varcharConverter.convert(Boolean.valueOf(false)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarchar("0", -1), varcharConverter.convert(Integer.valueOf(0)));
assertEquals("VarCharConverter", new HiveVarchar("1", -1), varcharConverter.convert(Integer.valueOf(1)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("0", -1)), varcharConverter.convert(Integer.valueOf(0)));
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("1", -1)), varcharConverter.convert(Integer.valueOf(1)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarchar("hive", -1), varcharConverter.convert(String.valueOf("hive")));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("hive", -1)), varcharConverter.convert(String.valueOf("hive")));
// Text
Converter textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("0"), textConverter.convert(Integer.valueOf(0)));
assertEquals("TextConverter", new Text("1"), textConverter.convert(Integer.valueOf(1)));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.writableBinaryObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("hive"), textConverter.convert(new BytesWritable(new byte[] { (byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e' })));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("hive"), textConverter.convert(new Text("hive")));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("hive"), textConverter.convert(new String("hive")));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("100.001"), textConverter.convert(HiveDecimal.create("100.001")));
assertEquals("TextConverter", null, textConverter.convert(null));
// Binary
Converter baConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
assertEquals("BAConverter", new BytesWritable(new byte[] { (byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e' }), baConverter.convert("hive"));
assertEquals("BAConverter", null, baConverter.convert(null));
baConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
assertEquals("BAConverter", new BytesWritable(new byte[] { (byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e' }), baConverter.convert(new Text("hive")));
assertEquals("BAConverter", null, baConverter.convert(null));
// Union
ArrayList<String> fieldNames = new ArrayList<String>();
fieldNames.add("firstInteger");
fieldNames.add("secondString");
fieldNames.add("thirdBoolean");
ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
ArrayList<String> fieldNames2 = new ArrayList<String>();
fieldNames2.add("firstString");
fieldNames2.add("secondInteger");
fieldNames2.add("thirdBoolean");
ArrayList<ObjectInspector> fieldObjectInspectors2 = new ArrayList<ObjectInspector>();
fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
Object convertedObject0 = unionConverter0.convert(new StandardUnion((byte) 0, 1));
StandardUnion expectedObject0 = new StandardUnion();
expectedObject0.setTag((byte) 0);
expectedObject0.setObject("1");
assertEquals(expectedObject0, convertedObject0);
Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
Object convertedObject1 = unionConverter1.convert(new StandardUnion((byte) 1, "1"));
StandardUnion expectedObject1 = new StandardUnion();
expectedObject1.setTag((byte) 1);
expectedObject1.setObject(1);
assertEquals(expectedObject1, convertedObject1);
Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
Object convertedObject2 = unionConverter2.convert(new StandardUnion((byte) 2, true));
StandardUnion expectedObject2 = new StandardUnion();
expectedObject2.setTag((byte) 2);
expectedObject2.setObject(true);
assertEquals(expectedObject2, convertedObject2);
// Union (extra fields)
ArrayList<String> fieldNamesExtra = new ArrayList<String>();
fieldNamesExtra.add("firstInteger");
fieldNamesExtra.add("secondString");
fieldNamesExtra.add("thirdBoolean");
ArrayList<ObjectInspector> fieldObjectInspectorsExtra = new ArrayList<ObjectInspector>();
fieldObjectInspectorsExtra.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
fieldObjectInspectorsExtra.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectorsExtra.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
ArrayList<String> fieldNamesExtra2 = new ArrayList<String>();
fieldNamesExtra2.add("firstString");
fieldNamesExtra2.add("secondInteger");
ArrayList<ObjectInspector> fieldObjectInspectorsExtra2 = new ArrayList<ObjectInspector>();
fieldObjectInspectorsExtra2.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectorsExtra2.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2));
Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnion((byte) 2, true));
StandardUnion expectedObjectExtra = new StandardUnion();
expectedObjectExtra.setTag((byte) -1);
expectedObjectExtra.setObject(null);
// we should get back null
assertEquals(expectedObjectExtra, convertedObjectExtra);
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.io.FloatWritable in project presto by prestodb.
the class OrcTester method decodeRecordReaderValue.
private static Object decodeRecordReaderValue(Type type, Object actualValue) {
if (actualValue instanceof OrcLazyObject) {
try {
actualValue = ((OrcLazyObject) actualValue).materialize();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
if (actualValue instanceof BooleanWritable) {
actualValue = ((BooleanWritable) actualValue).get();
} else if (actualValue instanceof ByteWritable) {
actualValue = ((ByteWritable) actualValue).get();
} else if (actualValue instanceof BytesWritable) {
actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
} else if (actualValue instanceof DateWritable) {
actualValue = new SqlDate(((DateWritable) actualValue).getDays());
} else if (actualValue instanceof DoubleWritable) {
actualValue = ((DoubleWritable) actualValue).get();
} else if (actualValue instanceof FloatWritable) {
actualValue = ((FloatWritable) actualValue).get();
} else if (actualValue instanceof IntWritable) {
actualValue = ((IntWritable) actualValue).get();
} else if (actualValue instanceof HiveCharWritable) {
actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
} else if (actualValue instanceof LongWritable) {
actualValue = ((LongWritable) actualValue).get();
} else if (actualValue instanceof ShortWritable) {
actualValue = ((ShortWritable) actualValue).get();
} else if (actualValue instanceof HiveDecimalWritable) {
DecimalType decimalType = (DecimalType) type;
HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
// writable messes with the scale so rescale the values to the Presto type
BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
} else if (actualValue instanceof Text) {
actualValue = actualValue.toString();
} else if (actualValue instanceof TimestampWritable) {
TimestampWritable timestamp = (TimestampWritable) actualValue;
actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
} else if (actualValue instanceof OrcStruct) {
List<Object> fields = new ArrayList<>();
OrcStruct structObject = (OrcStruct) actualValue;
for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
fields.add(OrcUtil.getFieldValue(structObject, fieldId));
}
actualValue = decodeRecordReaderStruct(type, fields);
} else if (actualValue instanceof com.facebook.hive.orc.OrcStruct) {
List<Object> fields = new ArrayList<>();
com.facebook.hive.orc.OrcStruct structObject = (com.facebook.hive.orc.OrcStruct) actualValue;
for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
fields.add(structObject.getFieldValue(fieldId));
}
actualValue = decodeRecordReaderStruct(type, fields);
} else if (actualValue instanceof List) {
actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
} else if (actualValue instanceof Map) {
actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
}
return actualValue;
}
use of org.apache.hadoop.io.FloatWritable in project druid by druid-io.
the class OrcStructConverterTest method testConvertRootFieldWithMapOfNullValuesReturningMapOfNulls.
@Test
public void testConvertRootFieldWithMapOfNullValuesReturningMapOfNulls() {
final TypeDescription mapType = TypeDescription.createMap(TypeDescription.createInt(), TypeDescription.createFloat());
final OrcMap<IntWritable, FloatWritable> map = new OrcMap<>(mapType);
IntStream.range(0, 3).forEach(i -> map.put(new IntWritable(i * 10), null));
final Map<Integer, Float> expectedResult = new HashMap<>();
for (Entry<IntWritable, FloatWritable> entry : map.entrySet()) {
expectedResult.put(entry.getKey().get(), null);
}
final OrcStructConverter converter = new OrcStructConverter(false);
assertConversion(converter, mapType, expectedResult, map);
}
use of org.apache.hadoop.io.FloatWritable in project druid by druid-io.
the class OrcStructConverterTest method testConvertRootFieldWithNonNullFloatReturningOriginalValue.
@Test
public void testConvertRootFieldWithNonNullFloatReturningOriginalValue() {
final OrcStructConverter converter = new OrcStructConverter(false);
assertConversion(converter, TypeDescription.createFloat(), 0.1f, new FloatWritable(0.1f));
}
Aggregations