Examples with StateStoreProvider io.prestosql.statestore.StateStoreProvider used on opensource projects

Example 11 with StateStoreProvider

use of io.prestosql.statestore.StateStoreProvider in project hetu-core by openlookeng.

the class TestSplitCacheStateInitializer method testInitializationTimedOut.

@Test
public void testInitializationTimedOut() {
    StateStoreProvider provider = mock(StateStoreProvider.class);
    // simulate behaviour wait until state store is ready
    when(provider.getStateStore()).thenReturn(null);
    SplitCacheMap splitCacheMap = createNew();
    assertFalse(splitCacheMap.cacheExists(table1QN));
    assertFalse(splitCacheMap.cacheExists(table2QN));
    AtomicReference<SplitCacheStateInitializer.InitializationStatus> status = new AtomicReference<>(SplitCacheStateInitializer.InitializationStatus.INITIALIZING);
    SplitCacheStateInitializer initializer = new SplitCacheStateInitializer(provider, splitCacheMap, new Duration(300, TimeUnit.MILLISECONDS), new Duration(1, TimeUnit.SECONDS), objectMapper, status);
    try {
        initializer.start();
        // sleep for sometime till local split cache map is initialized
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
        // ignore - nothing to do here
        }
    } finally {
        initializer.stop();
    }
    verify(provider, atLeastOnce()).getStateStore();
    assertEquals(status.get(), SplitCacheStateInitializer.InitializationStatus.FAILED);
}

Example 12 with StateStoreProvider

use of io.prestosql.statestore.StateStoreProvider in project hetu-core by openlookeng.

the class TestSplitCacheStateInitializer method testStateInitialization.

@Test
public void testStateInitialization() {
    StateStoreProvider provider = mock(StateStoreProvider.class);
    StateStore stateStore = mock(StateStore.class);
    when(stateStore.getName()).thenReturn("mock");
    Map<String, String> testStateMap = new HashMap<>();
    testStateMap.put(table2QN.toString(), "{\"fqTableName\":\"bogus_catalog.test_schema_2.test_table_2\",\"splitWorkersMap\":{\"SplitKey{catalog='bogus_catalog', schema='test_schema_2', table='test_table_2', start=0, end=30, lastModifiedTime=1589472398269, qualifiedTableName=bogus_catalog.test_schema_2.test_table_2, path='hdfs://hacluster/user/hive/warehouse/test_schema_2.db/test_table_2/a=23/000000_0'}\":\"838986e0-7484-4bf6-87b9-7dd0af6bf901\"},\"predicates\":[{\"columnMetadataTupleDomain\":{\"columnDomains\":[{\"column\":{\"name\":\"a\",\"type\":\"bigint\",\"nullable\":true,\"hidden\":false,\"properties\":{}},\"domain\":{\"values\":{\"@type\":\"sortable\",\"type\":\"bigint\",\"ranges\":[{\"low\":{\"type\":\"bigint\",\"valueBlock\":\"CgAAAExPTkdfQVJSQVkBAAAAABcAAAAAAAAA\",\"bound\":\"EXACTLY\"},\"high\":{\"type\":\"bigint\",\"valueBlock\":\"CgAAAExPTkdfQVJSQVkBAAAAABcAAAAAAAAA\",\"bound\":\"EXACTLY\"}}]},\"nullAllowed\":false}}]},\"cachePredicateString\":\"a = 23\"}],\"lastUpdated\":\"2020-05-14T11:06:38.968\"}");
    testStateMap.put(table1QN.toString(), "{\"fqTableName\":\"bogus_catalog.test_schema_1.test_table_1\",\"splitWorkersMap\":{\"SplitKey{catalog='bogus_catalog', schema='test_schema_1', table='test_table_1', start=0, end=10, lastModifiedTime=1589472398267, qualifiedTableName=bogus_catalog.test_schema_1.test_table_1, path='hdfs://hacluster/user/hive/warehouse/test_schema_1.db/test_table_1/a=23/000000_0'}\":\"838986e0-7484-4bf6-87b9-7dd0af6bf901\",\"SplitKey{catalog='bogus_catalog', schema='test_schema_1', table='test_table_1', start=0, end=20, lastModifiedTime=1589472398269, qualifiedTableName=bogus_catalog.test_schema_1.test_table_1, path='hdfs://hacluster/user/hive/warehouse/test_schema_1.db/test_table_1/b=88/000010_0'}\":\"838986e0-7484-4bf6-87b9-7dd0af6bf902\"},\"predicates\":[{\"columnMetadataTupleDomain\":{\"columnDomains\":[{\"column\":{\"name\":\"a\",\"type\":\"bigint\",\"nullable\":true,\"hidden\":false,\"properties\":{}},\"domain\":{\"values\":{\"@type\":\"sortable\",\"type\":\"bigint\",\"ranges\":[{\"low\":{\"type\":\"bigint\",\"valueBlock\":\"CgAAAExPTkdfQVJSQVkBAAAAABcAAAAAAAAA\",\"bound\":\"EXACTLY\"},\"high\":{\"type\":\"bigint\",\"valueBlock\":\"CgAAAExPTkdfQVJSQVkBAAAAABcAAAAAAAAA\",\"bound\":\"EXACTLY\"}}]},\"nullAllowed\":false}}]},\"cachePredicateString\":\"a = 23\"},{\"columnMetadataTupleDomain\":{\"columnDomains\":[{\"column\":{\"name\":\"b\",\"type\":\"bigint\",\"nullable\":true,\"hidden\":false,\"properties\":{}},\"domain\":{\"values\":{\"@type\":\"sortable\",\"type\":\"bigint\",\"ranges\":[{\"low\":{\"type\":\"bigint\",\"valueBlock\":\"CgAAAExPTkdfQVJSQVkBAAAAAFgAAAAAAAAA\",\"bound\":\"EXACTLY\"},\"high\":{\"type\":\"bigint\",\"valueBlock\":\"CgAAAExPTkdfQVJSQVkBAAAAAFgAAAAAAAAA\",\"bound\":\"EXACTLY\"}}]},\"nullAllowed\":false}}]},\"cachePredicateString\":\"b = 88\"}],\"lastUpdated\":\"2020-05-14T11:06:38.968\"}");
    when(stateStore.getStateCollection(StateStoreConstants.SPLIT_CACHE_METADATA_NAME)).thenReturn(new MockStateMap<>(StateStoreConstants.SPLIT_CACHE_METADATA_NAME, testStateMap));
    // simulate behaviour wait until state store is ready
    when(provider.getStateStore()).thenReturn(null).thenReturn(stateStore);
    SplitCacheMap splitCacheMap = createNew();
    assertFalse(splitCacheMap.cacheExists(table1QN));
    assertFalse(splitCacheMap.cacheExists(table2QN));
    AtomicReference<SplitCacheStateInitializer.InitializationStatus> status = new AtomicReference<>(SplitCacheStateInitializer.InitializationStatus.INITIALIZING);
    SplitCacheStateInitializer initializer = new SplitCacheStateInitializer(provider, splitCacheMap, new Duration(100, TimeUnit.MILLISECONDS), new Duration(60, TimeUnit.SECONDS), objectMapper, status);
    try {
        initializer.start();
        // sleep for sometime till local split cache map is initialized
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        // ignore - nothing to do here
        }
        verify(provider, atLeastOnce()).getStateStore();
        assertEquals(status.get(), SplitCacheStateInitializer.InitializationStatus.COMPLETED);
        assertTrue(splitCacheMap.cacheExists(table1QN));
        assertTrue(splitCacheMap.getCachedNodeId(table1SplitKey1).map(workerNode.getNodeIdentifier()::equals).orElse(false));
        assertTrue(splitCacheMap.getCachedNodeId(table1SplitKey2).map(workerNode2.getNodeIdentifier()::equals).orElse(false));
        assertTrue(splitCacheMap.cacheExists(table2QN));
        assertTrue(splitCacheMap.getCachedNodeId(table2SplitKey1).map(workerNode.getNodeIdentifier()::equals).orElse(false));
    } finally {
        initializer.stop();
    }
}

Example 13 with StateStoreProvider

use of io.prestosql.statestore.StateStoreProvider in project hetu-core by openlookeng.

the class TestSplitCacheStateManager method testStartAndStopServicesWhenStateStoreEnabled.

@Test
public void testStartAndStopServicesWhenStateStoreEnabled() {
    StateStoreProvider provider = mock(StateStoreProvider.class);
    when(provider.getStateStore()).thenReturn(stateStore);
    PropertyService.setProperty(HetuConstant.SPLIT_CACHE_MAP_ENABLED, true);
    PropertyService.setProperty(HetuConstant.MULTI_COORDINATOR_ENABLED, true);
    PropertyService.setProperty(HetuConstant.SPLIT_CACHE_STATE_UPDATE_INTERVAL, new Duration(2, TimeUnit.SECONDS));
    SplitCacheMap splitCacheMap = createNew(true);
    SplitCacheStateManager manager = new SplitCacheStateManager(provider, metadata, splitCacheMap);
    manager.startStateServices();
    manager.stopStateServices();
    assertNotNull(SplitCacheMap.getInstance());
}

Example 14 with StateStoreProvider

use of io.prestosql.statestore.StateStoreProvider in project hetu-core by openlookeng.

the class TestSplitCacheStateUpdater method testStateUpdates.

@Test
public void testStateUpdates() throws InterruptedException, IOException {
    StateStoreProvider provider = mock(StateStoreProvider.class);
    StateStore stateStore = mock(StateStore.class);
    when(stateStore.getName()).thenReturn("mock");
    Map<String, String> testStateMap = new HashMap<>();
    when(stateStore.getStateCollection(StateStoreConstants.SPLIT_CACHE_METADATA_NAME)).thenReturn(new MockStateMap<>(StateStoreConstants.SPLIT_CACHE_METADATA_NAME, testStateMap));
    // simulate behaviour wait until state store is ready
    when(provider.getStateStore()).thenReturn(null).thenReturn(stateStore);
    SplitCacheMap splitCacheMap = createNew();
    assertTrue(testStateMap.isEmpty());
    AtomicReference<InitializationStatus> status = new AtomicReference<>(InitializationStatus.COMPLETED);
    SplitCacheStateUpdater updater = new SplitCacheStateUpdater(provider, splitCacheMap, new Duration(500, TimeUnit.MILLISECONDS), objectMapper, status);
    try {
        updater.start();
        splitCacheMap.addCache(table1QN, tupleDomainA, tupleDomainAPredicateString);
        splitCacheMap.addCachedNode(table1SplitKey1, workerNode.getNodeIdentifier());
        Thread.sleep(2000);
        assertTrue(testStateMap.containsKey(table1QN.toString()));
        assertEquals(testStateMap.size(), 1);
        TableCacheInfo deserTable1Cache = objectMapper.readerFor(TableCacheInfo.class).readValue(testStateMap.get(table1QN.toString()));
        assertEquals(deserTable1Cache, splitCacheMap.tableCacheInfoMap().get(table1QN.toString()));
        splitCacheMap.addCache(table1QN, tupleDomainB, tupleDomainBPredicateString);
        splitCacheMap.addCachedNode(table1SplitKey2, workerNode2.getNodeIdentifier());
        splitCacheMap.addCache(table2QN, tupleDomainA, tupleDomainAPredicateString);
        splitCacheMap.addCachedNode(table2SplitKey1, workerNode2.getNodeIdentifier());
        Thread.sleep(2000);
        assertEquals(testStateMap.size(), 2);
        deserTable1Cache = objectMapper.readerFor(TableCacheInfo.class).readValue(testStateMap.get(table1QN.toString()));
        assertEquals(deserTable1Cache, splitCacheMap.tableCacheInfoMap().get(table1QN.toString()));
        TableCacheInfo deserTable2Cache = objectMapper.readerFor(TableCacheInfo.class).readValue(testStateMap.get(table2QN.toString()));
        assertEquals(deserTable2Cache, splitCacheMap.tableCacheInfoMap().get(table2QN.toString()));
        splitCacheMap.dropCache(table1QN, Optional.empty());
        assertFalse(splitCacheMap.cacheExists(table1QN));
        Thread.sleep(2000);
        assertEquals(testStateMap.size(), 1);
        assertTrue(testStateMap.containsKey(table2QN.toString()));
    } finally {
        updater.stop();
    }
}

Example 15 with StateStoreProvider

use of io.prestosql.statestore.StateStoreProvider in project hetu-core by openlookeng.

the class TestSplitCacheStateUpdater method testUpdateWhenStateStillInitializing.

@Test
public void testUpdateWhenStateStillInitializing() throws InterruptedException {
    StateStoreProvider provider = mock(StateStoreProvider.class);
    StateStore stateStore = mock(StateStore.class);
    when(stateStore.getName()).thenReturn("mock");
    when(provider.getStateStore()).thenReturn(stateStore);
    AtomicReference<InitializationStatus> status = new AtomicReference<>(InitializationStatus.INITIALIZING);
    SplitCacheMap splitCacheMap = createNew();
    SplitCacheStateUpdater updater = new SplitCacheStateUpdater(provider, splitCacheMap, new Duration(300, TimeUnit.MILLISECONDS), objectMapper, status);
    try {
        updater.start();
        Thread.sleep(1000);
    } finally {
        updater.stop();
    }
    verify(provider, atLeast(1)).getStateStore();
    verify(stateStore, never()).getStateCollection(StateStoreConstants.SPLIT_CACHE_METADATA_NAME);
}