Examples with Seed io.prestosql.spi.seedstore.Seed used on opensource projects

Example 16 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class SeedStoreManager method addSeed.

private Collection<Seed> addSeed(SeedStoreSubType subType, Seed seed) throws IOException {
    int retryTimes = 0;
    long retryInterval = 0L;
    Collection<Seed> seeds = null;
    SeedStore seedStore = getSeedStore(subType);
    if (seedStore == null) {
        throw new PrestoException(SEED_STORE_FAILURE, "Seed store is null");
    }
    do {
        try {
            TimeUnit.MILLISECONDS.sleep(retryInterval);
            seeds = seedStore.add(Lists.newArrayList(seed));
        } catch (InterruptedException | RuntimeException e) {
            LOG.warn("add seed=%s failed: %s, will retry at times: %s", seed, e.getMessage(), retryTimes);
        } finally {
            retryTimes++;
            retryInterval += SEED_RETRY_INTERVAL;
        }
    } while (retryTimes <= SEED_RETRY_TIMES && (seeds == null || seeds.size() == 0));
    if (seeds == null || seeds.size() == 0) {
        throw new PrestoException(SEED_STORE_FAILURE, String.format(Locale.ROOT, "add seed=%s to seed store failed after retry:%d", seed.getLocation(), SEED_RETRY_TIMES));
    }
    return seeds;
}

Example 17 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class SeedStoreManager method getLatestSeedLocation.

/**
 * Get all seeds from seed store
 *
 * @return the latest seed location in the seed store - https or http
 * @throws IOException
 */
public String getLatestSeedLocation(SeedStoreSubType subType, boolean httpsRequired) throws IOException {
    SeedStore seedStore = getSeedStore(subType);
    if (seedStore == null) {
        throw new PrestoException(SEED_STORE_FAILURE, "Seed store is null");
    }
    Collection<Seed> seeds = seedStore.get();
    if (seeds.isEmpty()) {
        return null;
    }
    ArrayList<Seed> list = seeds.stream().filter(seed -> {
        if (httpsRequired) {
            return seed.getLocation().contains("https:");
        } else {
            return seed.getLocation().contains("http:");
        }
    }).sorted(Comparator.comparing(Seed::getTimestamp)).collect(toCollection(ArrayList::new));
    if (list.isEmpty()) {
        return null;
    } else {
        return list.get(list.size() - 1).getLocation();
    }
}

Example 18 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class TestStateStoreLauncherAndProvider method testLaunchAndFailure.

// Test Launcher
@Test
public void testLaunchAndFailure() throws Exception {
    Set<Seed> seeds = new HashSet<>();
    SeedStore mockSeedStore = mock(SeedStore.class);
    Seed mockSeed1 = mock(Seed.class);
    Seed mockSeed2 = mock(Seed.class);
    seeds.add(mockSeed1);
    seeds.add(mockSeed2);
    when(mockSeed1.getLocation()).thenReturn(LOCALHOST + ":" + PORT1);
    when(mockSeed2.getLocation()).thenReturn(LOCALHOST + ":" + PORT2);
    when(mockSeedStore.get()).thenReturn(seeds);
    SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
    when(mockSeedStoreManager.getSeedStore(SeedStoreSubType.HAZELCAST)).thenReturn(mockSeedStore);
    when(mockSeedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, LOCALHOST, true)).thenReturn(seeds);
    when(mockSeedStoreManager.getFileSystemClient()).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get("/")));
    InternalCommunicationConfig mockInternalCommunicationConfig = mock(InternalCommunicationConfig.class);
    HttpServerInfo mockHttpServerInfo = mock(HttpServerInfo.class);
    when(mockHttpServerInfo.getHttpsUri()).thenReturn(new URI("https://" + LOCALHOST + ":" + PORT1));
    when(mockInternalCommunicationConfig.isHttpsRequired()).thenReturn(true);
    EmbeddedStateStoreLauncher launcher = new EmbeddedStateStoreLauncher(mockSeedStoreManager, mockInternalCommunicationConfig, mockHttpServerInfo, new HetuConfig());
    StateStoreBootstrapper bootstrapper = new HazelcastStateStoreBootstrapper();
    launcher.addStateStoreBootstrapper(bootstrapper);
    launcher.launchStateStore();
    StateStore second = setupSecondInstance();
    // mock "remove" second instance from cluster (delete from seed store)
    seeds.remove(mockSeed2);
    when(mockSeed1.getLocation()).thenReturn(LOCALHOST + ":" + PORT1);
    when(mockSeedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, LOCALHOST, true)).thenReturn(seeds);
    ((HazelcastStateStore) second).shutdown();
    // Allow the first node to handle failure
    Thread.sleep(3000L);
}

Example 19 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class FileBasedSeedStoreOnYarn method remove.

@Override
public Set<Seed> remove(Collection<Seed> seeds) throws IOException {
    LOG.debug("FileBasedOnYarnSeedStore::remove() invoked.");
    Path lockPath = seedFileFolder.resolve(name);
    checkArgument(!lockPath.toString().contains("../"), "Lock path must be absolute and at user workspace " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    checkArgument(SecurePathWhiteList.isSecurePath(lockPath.toString()), "Lock path must be at user workspace " + SecurePathWhiteList.getSecurePathWhiteList().toString());
    Lock lock = new FileBasedLock(fs, lockPath);
    try {
        lock.lock();
        Set<Seed> outputs = new HashSet<>();
        if (fs.exists(seedFilePath)) {
            String json = loadFromFile(seedFilePath);
            List<FileBasedSeedOnYarn> existingSeeds = LIST_FILE_BASED_SEED_CODEC.fromJson(json);
            Set<FileBasedSeedOnYarn> latestSeeds = new HashSet<>(existingSeeds);
            latestSeeds.removeAll(seeds.stream().filter(s -> (s instanceof FileBasedSeedOnYarn)).map(s -> (FileBasedSeedOnYarn) s).collect(Collectors.toList()));
            String output = LIST_FILE_BASED_SEED_CODEC.toJson(ImmutableList.copyOf(latestSeeds));
            writeToFile(seedFilePath, output, true);
            outputs.addAll(latestSeeds);
        }
        return outputs;
    } catch (UncheckedIOException e) {
        throw new IOException(e);
    } finally {
        lock.unlock();
    }
}

Example 20 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class TestFileBasedSeedStore method testBasics.

/**
 * Test add, remove and overwrite
 *
 * @throws IOException IOException happens if filesystem error happens
 */
@Test
public void testBasics() throws IOException {
    String ip1 = "10.0.0.1";
    final long timestamp1 = 1000L;
    String ip2 = "10.0.0.2";
    final long timestamp2 = 2000L;
    final int resultSize = 2;
    Seed seed1 = new FileBasedSeed(ip1, timestamp1);
    Seed seed2 = new FileBasedSeed(ip2, timestamp2);
    // add operation
    seedStore.add(Lists.newArrayList(seed1, seed2));
    Set<Seed> results = seedStore.get();
    assertEquals(results.size(), resultSize);
    assertTrue(results.contains(seed1));
    assertTrue(results.contains(seed2));
    Seed seedResult1 = getSeed(results, ip1);
    Seed seedResult2 = getSeed(results, ip2);
    assertEquals(seedResult1.getTimestamp(), timestamp1);
    assertEquals(seedResult2.getTimestamp(), timestamp2);
    // remove operation
    seedStore.remove(Lists.newArrayList(seed1));
    results = seedStore.get();
    assertEquals(results.size(), 1);
    assertFalse(results.contains(seed1));
    assertTrue(results.contains(seed2));
    seedResult2 = getSeed(results, ip2);
    assertEquals(seedResult2.getTimestamp(), timestamp2);
    // overwrite operation
    final long updateTimestamp2 = 3000L;
    Seed seed2Update = new FileBasedSeed(ip2, updateTimestamp2);
    seedStore.add(Lists.newArrayList(seed2Update));
    results = seedStore.get();
    assertEquals(results.size(), 1);
    assertTrue(results.contains(seed2));
    seedResult2 = getSeed(results, ip2);
    assertEquals(seedResult2.getTimestamp(), updateTimestamp2);
}