Search in sources :

Example 21 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class TestFileBasedSeedStore method testSeedCreate.

/**
 * Test Seed Creation
 */
@Test
public void testSeedCreate() {
    Map<String, String> prop = new HashMap<>(0);
    String location = "120.0.0.1";
    String timestamp = "100";
    prop.put(Seed.LOCATION_PROPERTY_NAME, location);
    prop.put(Seed.TIMESTAMP_PROPERTY_NAME, timestamp);
    Seed seed = seedStore.create(prop);
    assertEquals(seed.getLocation(), location);
    assertEquals(String.valueOf(seed.getTimestamp()), timestamp);
}
Also used : HashMap(java.util.HashMap) Seed(io.prestosql.spi.seedstore.Seed) Test(org.testng.annotations.Test)

Example 22 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class FileBasedSeedStore method getSeeds.

private Set<Seed> getSeeds() throws IOException {
    Set<Seed> outputs = new HashSet<>(0);
    StringBuilder content = new StringBuilder(0);
    if (fs.exists(seedFilePath)) {
        try (BufferedReader br = new BufferedReader(new InputStreamReader(fs.newInputStream(seedFilePath), StandardCharsets.UTF_8))) {
            br.lines().forEach(content::append);
        }
    }
    Set<String> seedsStrings = Stream.of(content.toString().split(COMMA)).filter(e -> !e.isEmpty()).collect(Collectors.toSet());
    for (String str : seedsStrings) {
        try {
            outputs.add(FileBasedSeed.deserialize(str));
        } catch (ClassNotFoundException e) {
            LOG.error("Seed string: %s cannot be deserialized", str);
        }
    }
    return outputs;
}
Also used : OutputStream(java.io.OutputStream) SeedStore(io.prestosql.spi.seedstore.SeedStore) Logger(io.airlift.log.Logger) HetuFileSystemClient(io.prestosql.spi.filesystem.HetuFileSystemClient) Collection(java.util.Collection) Seed(io.prestosql.spi.seedstore.Seed) Set(java.util.Set) IOException(java.io.IOException) FileBasedLock(io.prestosql.spi.filesystem.FileBasedLock) InputStreamReader(java.io.InputStreamReader) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) UncheckedIOException(java.io.UncheckedIOException) HashSet(java.util.HashSet) Lock(java.util.concurrent.locks.Lock) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Stream(java.util.stream.Stream) Paths(java.nio.file.Paths) Map(java.util.Map) BufferedReader(java.io.BufferedReader) CREATE_NEW(java.nio.file.StandardOpenOption.CREATE_NEW) Path(java.nio.file.Path) SecurePathWhiteList(io.hetu.core.common.util.SecurePathWhiteList) InputStreamReader(java.io.InputStreamReader) Seed(io.prestosql.spi.seedstore.Seed) BufferedReader(java.io.BufferedReader) HashSet(java.util.HashSet)

Example 23 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class TestHetuMetastoreGlobalCache method setUpHazelcast.

@BeforeTest
private void setUpHazelcast() throws IOException {
    Set<Seed> seeds = new HashSet<>();
    SeedStore mockSeedStore = mock(SeedStore.class);
    Seed mockSeed = mock(Seed.class);
    seeds.add(mockSeed);
    SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
    when(mockSeedStoreManager.getSeedStore(any(SeedStoreSubType.class))).thenReturn(mockSeedStore);
    when(mockSeed.getLocation()).thenReturn(LOCALHOST + ":" + PORT3);
    when(mockSeedStore.get()).thenReturn(seeds);
    factory = new HazelcastStateStoreFactory();
    stateStoreProvider = new LocalStateStoreProvider(mockSeedStoreManager);
    stateStoreProvider.addStateStoreFactory(factory);
}
Also used : LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) Seed(io.prestosql.spi.seedstore.Seed) SeedStore(io.prestosql.spi.seedstore.SeedStore) HazelcastStateStoreFactory(io.hetu.core.statestore.hazelcast.HazelcastStateStoreFactory) HashSet(java.util.HashSet) SeedStoreSubType(io.prestosql.spi.seedstore.SeedStoreSubType) BeforeTest(org.testng.annotations.BeforeTest)

Example 24 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class TestSeedStoreManager method testClearExpiredSeed.

@Test
void testClearExpiredSeed() throws Exception {
    String location1 = "location1";
    String location2 = "location2";
    seedStoreManager.loadSeedStore();
    seedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, location1, false);
    seedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, location2, true);
    Assert.assertEquals(seedStoreManager.getAllSeeds(SeedStoreSubType.HAZELCAST).size(), 2);
    // wait 5 seconds, location1 expired since refreshable is not enabled
    Thread.sleep(5000);
    seedStoreManager.clearExpiredSeeds(SeedStoreSubType.HAZELCAST);
    Collection<Seed> result = seedStoreManager.getAllSeeds(SeedStoreSubType.HAZELCAST);
    Assert.assertEquals(result.size(), 1);
    Assert.assertFalse(result.stream().filter(s -> s.getLocation().equals(location1)).findAny().isPresent());
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) HetuFileSystemClient(io.prestosql.spi.filesystem.HetuFileSystemClient) SeedStoreSubType(io.prestosql.spi.seedstore.SeedStoreSubType) Test(org.testng.annotations.Test) HashMap(java.util.HashMap) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) Matchers.anyString(org.mockito.Matchers.anyString) HashSet(java.util.HashSet) BeforeTest(org.testng.annotations.BeforeTest) Map(java.util.Map) ImmutableSet(com.google.common.collect.ImmutableSet) SeedStore(io.prestosql.spi.seedstore.SeedStore) Collection(java.util.Collection) BeforeMethod(org.testng.annotations.BeforeMethod) FileWriter(java.io.FileWriter) Seed(io.prestosql.spi.seedstore.Seed) Set(java.util.Set) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) File(java.io.File) SeedStoreFactory(io.prestosql.spi.seedstore.SeedStoreFactory) Objects(java.util.Objects) Matchers.any(org.mockito.Matchers.any) HAZELCAST_DISCOVERY_TCPIP_SEEDS(io.prestosql.statestore.StateStoreConstants.HAZELCAST_DISCOVERY_TCPIP_SEEDS) Assert(io.prestosql.testing.assertions.Assert) HAZELCAST_DISCOVERY_TCPIP_PROFILE(io.prestosql.statestore.StateStoreConstants.HAZELCAST_DISCOVERY_TCPIP_PROFILE) Mockito.mock(org.mockito.Mockito.mock) Seed(io.prestosql.spi.seedstore.Seed) Matchers.anyString(org.mockito.Matchers.anyString) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 25 with Seed

use of io.prestosql.spi.seedstore.Seed in project hetu-core by openlookeng.

the class TestSeedStoreManager method testRemoveFromSeedStore.

@Test
void testRemoveFromSeedStore() throws Exception {
    String location1 = "location1";
    String location2 = "location2";
    String location3 = "location3";
    seedStoreManager.loadSeedStore();
    seedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, location1, false);
    seedStoreManager.addSeed(SeedStoreSubType.HAZELCAST, location2, true);
    Assert.assertEquals(seedStoreManager.getAllSeeds(SeedStoreSubType.HAZELCAST).size(), 2);
    seedStoreManager.removeSeed(SeedStoreSubType.HAZELCAST, location1);
    seedStoreManager.removeSeed(SeedStoreSubType.HAZELCAST, location3);
    Collection<Seed> result = seedStoreManager.getAllSeeds(SeedStoreSubType.HAZELCAST);
    Assert.assertEquals(result.size(), 1);
    Assert.assertFalse(result.stream().filter(s -> s.getLocation().equals(location1)).findAny().isPresent());
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) HetuFileSystemClient(io.prestosql.spi.filesystem.HetuFileSystemClient) SeedStoreSubType(io.prestosql.spi.seedstore.SeedStoreSubType) Test(org.testng.annotations.Test) HashMap(java.util.HashMap) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) Matchers.anyString(org.mockito.Matchers.anyString) HashSet(java.util.HashSet) BeforeTest(org.testng.annotations.BeforeTest) Map(java.util.Map) ImmutableSet(com.google.common.collect.ImmutableSet) SeedStore(io.prestosql.spi.seedstore.SeedStore) Collection(java.util.Collection) BeforeMethod(org.testng.annotations.BeforeMethod) FileWriter(java.io.FileWriter) Seed(io.prestosql.spi.seedstore.Seed) Set(java.util.Set) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) File(java.io.File) SeedStoreFactory(io.prestosql.spi.seedstore.SeedStoreFactory) Objects(java.util.Objects) Matchers.any(org.mockito.Matchers.any) HAZELCAST_DISCOVERY_TCPIP_SEEDS(io.prestosql.statestore.StateStoreConstants.HAZELCAST_DISCOVERY_TCPIP_SEEDS) Assert(io.prestosql.testing.assertions.Assert) HAZELCAST_DISCOVERY_TCPIP_PROFILE(io.prestosql.statestore.StateStoreConstants.HAZELCAST_DISCOVERY_TCPIP_PROFILE) Mockito.mock(org.mockito.Mockito.mock) Seed(io.prestosql.spi.seedstore.Seed) Matchers.anyString(org.mockito.Matchers.anyString) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Aggregations

Seed (io.prestosql.spi.seedstore.Seed)29 SeedStore (io.prestosql.spi.seedstore.SeedStore)21 HashSet (java.util.HashSet)20 IOException (java.io.IOException)15 Map (java.util.Map)12 PrestoException (io.prestosql.spi.PrestoException)11 Collection (java.util.Collection)11 HashMap (java.util.HashMap)10 HetuFileSystemClient (io.prestosql.spi.filesystem.HetuFileSystemClient)8 Set (java.util.Set)8 Test (org.testng.annotations.Test)8 Logger (io.airlift.log.Logger)7 SeedStoreSubType (io.prestosql.spi.seedstore.SeedStoreSubType)7 File (java.io.File)7 Collectors (java.util.stream.Collectors)7 BeforeTest (org.testng.annotations.BeforeTest)7 FileBasedLock (io.prestosql.spi.filesystem.FileBasedLock)6 SeedStoreFactory (io.prestosql.spi.seedstore.SeedStoreFactory)6 Paths (java.nio.file.Paths)6 Lock (java.util.concurrent.locks.Lock)6