Examples with Seed io.prestosql.spi.seedstore.Seed used on opensource projects

Example 11 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class FileBasedSeedStoreOnYarn method add.

@Override
public Set<Seed> add(Collection<Seed> seeds) throws IOException {
    LOG.debug("FileBasedOnYarnSeedStore::add() invoked.");
    Path lockPath = seedFileFolder.resolve(name);
    checkArgument(!lockPath.toString().contains("../"), "Lock path must be absolute and at user workspace " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    checkArgument(SecurePathWhiteList.isSecurePath(lockPath.toString()), "Lock path must be at user workspace " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    Lock lock = new FileBasedLock(fs, lockPath);
    try {
        lock.lock();
        Set<FileBasedSeedOnYarn> latestSeeds = new HashSet<>();
        // add all new seeds
        latestSeeds.addAll(seeds.stream().filter(s -> (s instanceof FileBasedSeedOnYarn)).map(s -> (FileBasedSeedOnYarn) s).collect(Collectors.toList()));
        // load existing seeds and filter out repeated seed compared to new seeds
        if (fs.exists(seedFilePath)) {
            String json = loadFromFile(seedFilePath);
            List<FileBasedSeedOnYarn> existingSeeds = LIST_FILE_BASED_SEED_CODEC.fromJson(json);
            latestSeeds.addAll(existingSeeds.stream().filter(s -> !latestSeeds.contains(s)).collect(Collectors.toList()));
        }
        String output = LIST_FILE_BASED_SEED_CODEC.toJson(ImmutableList.copyOf(latestSeeds));
        writeToFile(seedFilePath, output, true);
        return new HashSet<>(latestSeeds);
    } catch (UncheckedIOException e) {
        throw new IOException(e);
    } finally {
        lock.unlock();
    }
}

Example 12 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class FileBasedSeedStoreOnYarn method get.

@Override
public Set<Seed> get() throws IOException {
    LOG.debug("FileBasedOnYarnSeedStore::get() invoked.");
    try {
        Set<Seed> outputs = new HashSet<>();
        if (fs.exists(seedFilePath)) {
            String json = loadFromFile(seedFilePath);
            outputs.addAll(LIST_FILE_BASED_SEED_CODEC.fromJson(json));
        }
        return outputs;
    } catch (UncheckedIOException e) {
        throw new IOException(e);
    }
}

Example 13 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class FileBasedSeedStore method add.

@Override
public Set<Seed> add(Collection<Seed> seeds) throws IOException {
    Lock lock = new FileBasedLock(fs, seedDir.resolve(name));
    try {
        lock.lock();
        Set<String> writtens = new HashSet<>(0);
        Set<Seed> outputs = getSeeds();
        // overwrite all seeds
        outputs.removeAll(seeds);
        outputs.addAll(seeds);
        for (Seed seed : outputs) {
            writtens.add(seed.serialize());
        }
        writeToFile(seedFilePath, String.join(COMMA, writtens), true);
        return outputs;
    } catch (UncheckedIOException e) {
        throw new IOException(e);
    } finally {
        lock.unlock();
    }
}

Example 14 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class FileBasedSeedStore method remove.

@Override
public Set<Seed> remove(Collection<Seed> seeds) throws IOException {
    Lock lock = new FileBasedLock(fs, seedDir.resolve(name));
    try {
        lock.lock();
        Set<String> writtens = new HashSet<>(0);
        Set<Seed> outputs = getSeeds();
        outputs.removeAll(seeds);
        for (Seed seed : outputs) {
            writtens.add(seed.serialize());
        }
        writeToFile(seedFilePath, String.join(COMMA, writtens), true);
        return outputs;
    } catch (UncheckedIOException e) {
        throw new IOException(e);
    } finally {
        lock.unlock();
    }
}

Example 15 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class SeedStoreManager method clearExpiredSeeds.

/**
 * Clear expired seed in the seed store
 *
 * @throws IOException
 */
public void clearExpiredSeeds(SeedStoreSubType subType) throws IOException {
    SeedStore seedStore = getSeedStore(subType);
    if (seedStore == null) {
        throw new PrestoException(SEED_STORE_FAILURE, "Seed store is null");
    }
    try {
        Collection<Seed> expiredSeeds = seedStore.get().stream().filter(s -> (System.currentTimeMillis() - s.getTimestamp() > seedHeartBeatTimeout)).collect(Collectors.toList());
        if (expiredSeeds.size() > 0) {
            LOG.info("Expired seeds=%s will be cleared", expiredSeeds);
            seedStore.remove(expiredSeeds);
        }
    } catch (RuntimeException e) {
        LOG.warn("clearExpiredSeed failed with following message: %s", e.getMessage());
    }
}