Examples with Kryo com.esotericsoftware.kryo.Kryo used on opensource projects

Example 11 with Kryo

use of com.esotericsoftware.kryo.Kryo in project hive by apache.

the class FSStatsAggregator method connect.

@Override
public boolean connect(StatsCollectionContext scc) {
    List<String> statsDirs = scc.getStatsTmpDirs();
    assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs;
    Path statsDir = new Path(statsDirs.get(0));
    LOG.debug("About to read stats from : " + statsDir);
    statsMap = new HashMap<String, Map<String, String>>();
    try {
        fs = statsDir.getFileSystem(scc.getHiveConf());
        statsList = new ArrayList<Map<String, Map<String, String>>>();
        FileStatus[] status = fs.listStatus(statsDir, new PathFilter() {

            @Override
            public boolean accept(Path file) {
                return file.getName().startsWith(StatsSetupConst.STATS_FILE_PREFIX);
            }
        });
        for (FileStatus file : status) {
            Input in = new Input(fs.open(file.getPath()));
            Kryo kryo = SerializationUtilities.borrowKryo();
            try {
                statsMap = kryo.readObject(in, statsMap.getClass());
            } finally {
                SerializationUtilities.releaseKryo(kryo);
            }
            LOG.info("Read stats : " + statsMap);
            statsList.add(statsMap);
            in.close();
        }
        return true;
    } catch (IOException e) {
        LOG.error("Failed to read stats from filesystem ", e);
        return false;
    }
}

Example 12 with Kryo

use of com.esotericsoftware.kryo.Kryo in project hive by apache.

the class FSStatsPublisher method closeConnection.

@Override
public boolean closeConnection(StatsCollectionContext context) {
    List<String> statsDirs = context.getStatsTmpDirs();
    assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs;
    Path statsDir = new Path(statsDirs.get(0));
    try {
        Path statsFile = null;
        if (context.getIndexForTezUnion() != -1) {
            statsFile = new Path(statsDir, StatsSetupConst.STATS_FILE_PREFIX + conf.getInt("mapred.task.partition", 0) + "_" + context.getIndexForTezUnion());
        } else {
            statsFile = new Path(statsDir, StatsSetupConst.STATS_FILE_PREFIX + conf.getInt("mapred.task.partition", 0));
        }
        LOG.debug("About to create stats file for this task : " + statsFile);
        Output output = new Output(statsFile.getFileSystem(conf).create(statsFile, true));
        LOG.debug("Created file : " + statsFile);
        LOG.debug("Writing stats in it : " + statsMap);
        Kryo kryo = SerializationUtilities.borrowKryo();
        try {
            kryo.writeObject(output, statsMap);
        } finally {
            SerializationUtilities.releaseKryo(kryo);
        }
        output.close();
        return true;
    } catch (IOException e) {
        LOG.error("Failed to persist stats on filesystem", e);
        return false;
    }
}

Example 13 with Kryo

use of com.esotericsoftware.kryo.Kryo in project storm by apache.

the class SerializationFactoryTest method test_registers_when_valid_class_name.

@Test
public void test_registers_when_valid_class_name() {
    Class arbitraryClass = BlowfishTupleSerializer.class;
    String secretKey = "0123456789abcdef";
    Map conf = Utils.readDefaultConfig();
    conf.put(Config.TOPOLOGY_TUPLE_SERIALIZER, arbitraryClass.getName());
    conf.put(BlowfishTupleSerializer.SECRET_KEY, secretKey);
    Kryo kryo = SerializationFactory.getKryo(conf);
    Assert.assertEquals(arbitraryClass, kryo.getSerializer(ListDelegate.class).getClass());
}

Example 14 with Kryo

use of com.esotericsoftware.kryo.Kryo in project storm by apache.

the class SerializationFactoryTest method test_registers_default_when_not_in_conf.

@Test
public void test_registers_default_when_not_in_conf() throws ClassNotFoundException {
    Map conf = Utils.readDefaultConfig();
    String className = (String) conf.get(Config.TOPOLOGY_TUPLE_SERIALIZER);
    Class configuredClass = Class.forName(className);
    Kryo kryo = SerializationFactory.getKryo(conf);
    Assert.assertEquals(configuredClass, kryo.getSerializer(ListDelegate.class).getClass());
}

Example 15 with Kryo

use of com.esotericsoftware.kryo.Kryo in project storm by nathanmarz.

the class SerializationFactory method getKryo.

public static Kryo getKryo(Map conf) {
    IKryoFactory kryoFactory = (IKryoFactory) Utils.newInstance((String) conf.get(Config.TOPOLOGY_KRYO_FACTORY));
    Kryo k = kryoFactory.getKryo(conf);
    k.register(byte[].class);
    /* tuple payload serializer is specified via configuration */
    String payloadSerializerName = (String) conf.get(Config.TOPOLOGY_TUPLE_SERIALIZER);
    try {
        Class serializerClass = Class.forName(payloadSerializerName);
        Serializer serializer = resolveSerializerInstance(k, ListDelegate.class, serializerClass, conf);
        k.register(ListDelegate.class, serializer);
    } catch (ClassNotFoundException ex) {
        throw new RuntimeException(ex);
    }
    k.register(ArrayList.class, new ArrayListSerializer());
    k.register(HashMap.class, new HashMapSerializer());
    k.register(HashSet.class, new HashSetSerializer());
    k.register(BigInteger.class, new BigIntegerSerializer());
    k.register(TransactionAttempt.class);
    k.register(Values.class);
    k.register(backtype.storm.metric.api.IMetricsConsumer.DataPoint.class);
    k.register(backtype.storm.metric.api.IMetricsConsumer.TaskInfo.class);
    try {
        JavaBridge.registerPrimitives(k);
        JavaBridge.registerCollections(k);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Map<String, String> registrations = normalizeKryoRegister(conf);
    kryoFactory.preRegister(k, conf);
    boolean skipMissing = (Boolean) conf.get(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS);
    for (String klassName : registrations.keySet()) {
        String serializerClassName = registrations.get(klassName);
        try {
            Class klass = Class.forName(klassName);
            Class serializerClass = null;
            if (serializerClassName != null)
                serializerClass = Class.forName(serializerClassName);
            if (serializerClass == null) {
                k.register(klass);
            } else {
                k.register(klass, resolveSerializerInstance(k, klass, serializerClass, conf));
            }
        } catch (ClassNotFoundException e) {
            if (skipMissing) {
                LOG.info("Could not find serialization or class for " + serializerClassName + ". Skipping registration...");
            } else {
                throw new RuntimeException(e);
            }
        }
    }
    kryoFactory.postRegister(k, conf);
    if (conf.get(Config.TOPOLOGY_KRYO_DECORATORS) != null) {
        for (String klassName : (List<String>) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)) {
            try {
                Class klass = Class.forName(klassName);
                IKryoDecorator decorator = (IKryoDecorator) klass.newInstance();
                decorator.decorate(k);
            } catch (ClassNotFoundException e) {
                if (skipMissing) {
                    LOG.info("Could not find kryo decorator named " + klassName + ". Skipping registration...");
                } else {
                    throw new RuntimeException(e);
                }
            } catch (InstantiationException e) {
                throw new RuntimeException(e);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    }
    kryoFactory.postDecorate(k, conf);
    return k;
}