Examples with SeedStore io.prestosql.spi.seedstore.SeedStore used on opensource projects

Example 11 with SeedStore

use of io.prestosql.spi.seedstore.SeedStore in project hetu-core by openlookeng.

the class TestHazelcastStateStoreFactory method testTcpipCreate.

/**
 * Test state store creation with tcp-ip mode
 *
 * @throws IOException IOException thrown if seedStore read write errors happen
 */
@Test
public void testTcpipCreate() throws IOException {
    HazelcastStateStoreFactory factory = new HazelcastStateStoreFactory();
    assertEquals(factory.getName(), HAZELCAST);
    SeedStoreFactory seedStoreFactory = new FileBasedSeedStoreFactory();
    assertEquals(seedStoreFactory.getName(), "filebased");
    Map<String, String> properties = new HashMap<>(0);
    properties.put(STATE_STORE_CLUSTER_CONFIG_NAME, TEST_CLUSTER_NAME);
    properties.put(DISCOVERY_MODE_CONFIG_NAME, DISCOVERY_MODE_TCPIP);
    SeedStore seedStore = mock(SeedStore.class);
    Seed seed = new FileBasedSeed(MEMBER_ADDRESS, 0);
    when(seedStore.get()).thenReturn(ImmutableList.of(seed));
    setupHazelcastInstance();
    StateStore stateStore = factory.create(TEST_STATE_STORE_NAME, seedStore, properties);
    assertEquals(stateStore.getName(), TEST_STATE_STORE_NAME);
    StateCollection collection = stateStore.createStateCollection("test", StateCollection.Type.MAP);
    ((StateMap<String, String>) collection).put(TEST_KEY, TEST_VALUE);
    assertEquals(collection.size(), 1);
    assertEquals(((StateMap<String, String>) collection).get(TEST_KEY), TEST_VALUE);
    ((HazelcastStateStore) stateStore).shutdown();
}

Example 12 with SeedStore

use of io.prestosql.spi.seedstore.SeedStore in project hetu-core by openlookeng.

the class SeedStoreManager method clearExpiredSeeds.

/**
 * Clear expired seed in the seed store
 *
 * @throws IOException
 */
public void clearExpiredSeeds(SeedStoreSubType subType) throws IOException {
    SeedStore seedStore = getSeedStore(subType);
    if (seedStore == null) {
        throw new PrestoException(SEED_STORE_FAILURE, "Seed store is null");
    }
    try {
        Collection<Seed> expiredSeeds = seedStore.get().stream().filter(s -> (System.currentTimeMillis() - s.getTimestamp() > seedHeartBeatTimeout)).collect(Collectors.toList());
        if (expiredSeeds.size() > 0) {
            LOG.info("Expired seeds=%s will be cleared", expiredSeeds);
            seedStore.remove(expiredSeeds);
        }
    } catch (RuntimeException e) {
        LOG.warn("clearExpiredSeed failed with following message: %s", e.getMessage());
    }
}

Example 13 with SeedStore

use of io.prestosql.spi.seedstore.SeedStore in project hetu-core by openlookeng.

the class SeedStoreManager method addSeed.

private Collection<Seed> addSeed(SeedStoreSubType subType, Seed seed) throws IOException {
    int retryTimes = 0;
    long retryInterval = 0L;
    Collection<Seed> seeds = null;
    SeedStore seedStore = getSeedStore(subType);
    if (seedStore == null) {
        throw new PrestoException(SEED_STORE_FAILURE, "Seed store is null");
    }
    do {
        try {
            TimeUnit.MILLISECONDS.sleep(retryInterval);
            seeds = seedStore.add(Lists.newArrayList(seed));
        } catch (InterruptedException | RuntimeException e) {
            LOG.warn("add seed=%s failed: %s, will retry at times: %s", seed, e.getMessage(), retryTimes);
        } finally {
            retryTimes++;
            retryInterval += SEED_RETRY_INTERVAL;
        }
    } while (retryTimes <= SEED_RETRY_TIMES && (seeds == null || seeds.size() == 0));
    if (seeds == null || seeds.size() == 0) {
        throw new PrestoException(SEED_STORE_FAILURE, String.format(Locale.ROOT, "add seed=%s to seed store failed after retry:%d", seed.getLocation(), SEED_RETRY_TIMES));
    }
    return seeds;
}

Example 14 with SeedStore

use of io.prestosql.spi.seedstore.SeedStore in project hetu-core by openlookeng.

the class SeedStoreManager method getLatestSeedLocation.

/**
 * Get all seeds from seed store
 *
 * @return the latest seed location in the seed store - https or http
 * @throws IOException
 */
public String getLatestSeedLocation(SeedStoreSubType subType, boolean httpsRequired) throws IOException {
    SeedStore seedStore = getSeedStore(subType);
    if (seedStore == null) {
        throw new PrestoException(SEED_STORE_FAILURE, "Seed store is null");
    }
    Collection<Seed> seeds = seedStore.get();
    if (seeds.isEmpty()) {
        return null;
    }
    ArrayList<Seed> list = seeds.stream().filter(seed -> {
        if (httpsRequired) {
            return seed.getLocation().contains("https:");
        } else {
            return seed.getLocation().contains("http:");
        }
    }).sorted(Comparator.comparing(Seed::getTimestamp)).collect(toCollection(ArrayList::new));
    if (list.isEmpty()) {
        return null;
    } else {
        return list.get(list.size() - 1).getLocation();
    }
}

Example 15 with SeedStore

use of io.prestosql.spi.seedstore.SeedStore in project hetu-core by openlookeng.

the class TestStateStoreLauncherAndProvider method testLaunchAndFailure.

// Test Launcher
@Test
public void testLaunchAndFailure() throws Exception {
    Set<Seed> seeds = new HashSet<>();
    SeedStore mockSeedStore = mock(SeedStore.class);
    Seed mockSeed1 = mock(Seed.class);
    Seed mockSeed2 = mock(Seed.class);
    seeds.add(mockSeed1);
    seeds.add(mockSeed2);
    when(mockSeed1.getLocation()).thenReturn(LOCALHOST + ":" + PORT1);
    when(mockSeed2.getLocation()).thenReturn(LOCALHOST + ":" + PORT2);
    when(mockSeedStore.get()).thenReturn(seeds);
    SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
    when(mockSeedStoreManager.getSeedStore(SeedStoreSubType.HAZELCAST)).thenReturn(mockSeedStore);
    when(mockSeedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, LOCALHOST, true)).thenReturn(seeds);
    when(mockSeedStoreManager.getFileSystemClient()).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get("/")));
    InternalCommunicationConfig mockInternalCommunicationConfig = mock(InternalCommunicationConfig.class);
    HttpServerInfo mockHttpServerInfo = mock(HttpServerInfo.class);
    when(mockHttpServerInfo.getHttpsUri()).thenReturn(new URI("https://" + LOCALHOST + ":" + PORT1));
    when(mockInternalCommunicationConfig.isHttpsRequired()).thenReturn(true);
    EmbeddedStateStoreLauncher launcher = new EmbeddedStateStoreLauncher(mockSeedStoreManager, mockInternalCommunicationConfig, mockHttpServerInfo, new HetuConfig());
    StateStoreBootstrapper bootstrapper = new HazelcastStateStoreBootstrapper();
    launcher.addStateStoreBootstrapper(bootstrapper);
    launcher.launchStateStore();
    StateStore second = setupSecondInstance();
    // mock "remove" second instance from cluster (delete from seed store)
    seeds.remove(mockSeed2);
    when(mockSeed1.getLocation()).thenReturn(LOCALHOST + ":" + PORT1);
    when(mockSeedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, LOCALHOST, true)).thenReturn(seeds);
    ((HazelcastStateStore) second).shutdown();
    // Allow the first node to handle failure
    Thread.sleep(3000L);
}