Examples with SeedStoreFactory io.prestosql.spi.seedstore.SeedStoreFactory used on opensource projects

Example 1 with SeedStoreFactory

use of io.prestosql.spi.seedstore.SeedStoreFactory in project hetu-core by openlookeng.

the class PluginManager method installPlugin.

public void installPlugin(Plugin plugin) {
    for (BlockEncoding blockEncoding : plugin.getBlockEncodings()) {
        log.info("Registering block encoding %s", blockEncoding.getName());
        metadataManager.getFunctionAndTypeManager().addBlockEncoding(blockEncoding);
    }
    for (Type type : plugin.getTypes()) {
        log.info("Registering type %s", type.getTypeSignature());
        metadataManager.getFunctionAndTypeManager().addType(type);
    }
    for (ParametricType parametricType : plugin.getParametricTypes()) {
        log.info("Registering parametric type %s", parametricType.getName());
        metadataManager.getFunctionAndTypeManager().addParametricType(parametricType);
    }
    for (ConnectorFactory connectorFactory : plugin.getConnectorFactories()) {
        log.info("Registering connector %s", connectorFactory.getName());
        connectorManager.addConnectorFactory(connectorFactory);
        ConnectorCache.addCatalogConfig(plugin, connectorFactory.getName());
    }
    for (SessionPropertyConfigurationManagerFactory sessionConfigFactory : plugin.getSessionPropertyConfigurationManagerFactories()) {
        log.info("Registering session property configuration manager %s", sessionConfigFactory.getName());
        sessionPropertyDefaults.addConfigurationManagerFactory(sessionConfigFactory);
    }
    for (FunctionNamespaceManagerFactory functionNamespaceManagerFactory : plugin.getFunctionNamespaceManagerFactories()) {
        log.info("Registering function namespace manager %s", functionNamespaceManagerFactory.getName());
        metadataManager.getFunctionAndTypeManager().addFunctionNamespaceFactory(functionNamespaceManagerFactory);
    }
    for (ResourceGroupConfigurationManagerFactory configurationManagerFactory : plugin.getResourceGroupConfigurationManagerFactories()) {
        log.info("Registering resource group configuration manager %s", configurationManagerFactory.getName());
        resourceGroupManager.addConfigurationManagerFactory(configurationManagerFactory);
    }
    for (SystemAccessControlFactory accessControlFactory : plugin.getSystemAccessControlFactories()) {
        log.info("Registering system access control %s", accessControlFactory.getName());
        accessControlManager.addSystemAccessControlFactory(accessControlFactory);
    }
    for (PasswordAuthenticatorFactory authenticatorFactory : plugin.getPasswordAuthenticatorFactories()) {
        log.info("Registering password authenticator %s", authenticatorFactory.getName());
        passwordAuthenticatorManager.addPasswordAuthenticatorFactory(authenticatorFactory);
    }
    for (EventListenerFactory eventListenerFactory : plugin.getEventListenerFactories()) {
        log.info("Registering event listener %s", eventListenerFactory.getName());
        eventListenerManager.addEventListenerFactory(eventListenerFactory);
    }
    for (GroupProviderFactory groupProviderFactory : plugin.getGroupProviderFactories()) {
        log.info("Registering group provider %s", groupProviderFactory.getName());
        groupProviderManager.addGroupProviderFactory(groupProviderFactory);
    }
    // Install StateStorePlugin
    for (StateStoreBootstrapper bootstrapper : plugin.getStateStoreBootstrappers()) {
        log.info("Registering  state store bootstrapper");
        stateStoreLauncher.addStateStoreBootstrapper(bootstrapper);
    }
    for (StateStoreFactory stateStoreFactory : plugin.getStateStoreFactories()) {
        log.info("Registering state store %s", stateStoreFactory.getName());
        localStateStoreProvider.addStateStoreFactory(stateStoreFactory);
    }
    for (SeedStoreFactory seedStoreFactory : plugin.getSeedStoreFactories()) {
        log.info("Registering seed store %s", seedStoreFactory.getName());
        seedStoreManager.addSeedStoreFactory(seedStoreFactory);
    }
    for (CubeProvider cubeProvider : plugin.getCubeProviders()) {
        log.info("Registering cube provider %s", cubeProvider.getName());
        cubeManager.addCubeProvider(cubeProvider);
    }
    for (HetuFileSystemClientFactory fileSystemClientFactory : plugin.getFileSystemClientFactory()) {
        log.info("Registering file system provider %s", fileSystemClientFactory.getName());
        fileSystemClientManager.addFileSystemClientFactories(fileSystemClientFactory);
    }
    for (HetuMetaStoreFactory hetuMetaStoreFactory : plugin.getHetuMetaStoreFactories()) {
        log.info("Registering hetu metastore %s", hetuMetaStoreFactory.getName());
        hetuMetaStoreManager.addHetuMetaStoreFactory(hetuMetaStoreFactory);
    }
    for (IndexFactory indexFactory : plugin.getIndexFactories()) {
        log.info("Loading index factory");
        heuristicIndexerManager.loadIndexFactories(indexFactory);
    }
    installFunctionsPlugin(plugin);
}

Example 2 with SeedStoreFactory

use of io.prestosql.spi.seedstore.SeedStoreFactory in project hetu-core by openlookeng.

the class SeedStoreManager method loadSeedStore.

/**
 * Use the registered SeedStoreFactory to load SeedStore
 *
 * @throws IOException exception when fail to create SeedStore
 */
public void loadSeedStore() throws IOException {
    // initialize variables
    isSeedStoreOnYarnEnabled = SEED_STORE_ON_YARN_ENABLED_DEFAULT_VALUE;
    seedStoreType = SEED_STORE_TYPE_DEFAULT_VALUE;
    clusterName = SEED_STORE_CLUSTER_DEFAULT_VALUE;
    filesystemProfile = SEED_STORE_FILESYSTEM_PROFILE_DEFAULT_VALUE;
    seedHeartBeat = SEED_STORE_SEED_HEARTBEAT_DEFAULT_VALUE;
    seedHeartBeatTimeout = SEED_STORE_SEED_HEARTBEAT_TIMEOUT_DEFAULT_VALUE;
    Map<String, String> config = new HashMap<>();
    // load seed store configuration
    if (SEED_STORE_CONFIGURATION.exists()) {
        Map<String, String> seedStoreProperties = new HashMap<>(loadPropertiesFrom(SEED_STORE_CONFIGURATION.getPath()));
        String propertyValue = seedStoreProperties.get(SEED_STORE_ON_YARN_PROPERTY_NAME);
        if (propertyValue != null) {
            isSeedStoreOnYarnEnabled = Boolean.parseBoolean(propertyValue);
        }
        seedStoreType = seedStoreProperties.getOrDefault(SEED_STORE_TYPE_PROPERTY_NAME, seedStoreType);
        clusterName = seedStoreProperties.getOrDefault(SEED_STORE_CLUSTER_PROPERTY_NAME, clusterName);
        filesystemProfile = seedStoreProperties.getOrDefault(SEED_STORE_FILESYSTEM_PROFILE, filesystemProfile);
        propertyValue = seedStoreProperties.get(SEED_STORE_SEED_HEARTBEAT_PROPERTY_NAME);
        if (propertyValue != null) {
            seedHeartBeat = Long.parseLong(propertyValue);
        }
        propertyValue = seedStoreProperties.get(SEED_STORE_SEED_HEARTBEAT_TIMEOUT_PROPERTY_NAME);
        if (propertyValue != null) {
            seedHeartBeatTimeout = Long.parseLong(propertyValue);
        }
        if (seedHeartBeat > seedHeartBeatTimeout) {
            throw new InvalidParameterException(format("The value of %s cannot be greater than the value of %s in the property file", SEED_STORE_SEED_HEARTBEAT_PROPERTY_NAME, SEED_STORE_SEED_HEARTBEAT_TIMEOUT_PROPERTY_NAME));
        }
        config.putAll(seedStoreProperties);
        if (isSeedStoreOnYarnEnabled) {
            // create seed store
            SeedStoreFactory seedStoreFactory = seedStoreFactories.get(seedStoreType);
            checkState(seedStoreFactory != null, "SeedStoreFactory %s is not registered", seedStoreFactory);
            if (fileSystemClient == null) {
                fileSystemClient = fileSystemClientManager.getFileSystemClient(filesystemProfile, Paths.get("/"));
            }
            LOG.info("-- Loading seed store on-yarn --");
            try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(seedStoreFactory.getClass().getClassLoader())) {
                seedStoreOnYarn = seedStoreFactory.create(clusterName, SeedStoreSubType.ON_YARN, fileSystemClient, ImmutableMap.copyOf(config));
            }
        }
    }
    // load state store config if exist
    if (STATE_STORE_CONFIGURATION.exists()) {
        Map<String, String> stateStoreProperties = new HashMap<>(loadPropertiesFrom(STATE_STORE_CONFIGURATION.getPath()));
        // for now, seed store is started only if tcp-ip mode enabled and tcp-ip.seeds is not set
        String discoveryMode = stateStoreProperties.get(StateStoreConstants.DISCOVERY_MODE_PROPERTY_NAME);
        isSeedStoreHazelcastEnabled = discoveryMode != null && discoveryMode.equals(StateStoreConstants.DISCOVERY_MODE_TCPIP) && stateStoreProperties.get(StateStoreConstants.HAZELCAST_DISCOVERY_TCPIP_SEEDS) == null;
        if (isSeedStoreHazelcastEnabled) {
            // create seed store
            SeedStoreFactory seedStoreFactory = seedStoreFactories.get(seedStoreType);
            checkState(seedStoreFactory != null, "SeedStoreFactory %s is not registered", seedStoreFactory);
            if (fileSystemClient == null) {
                String stateStoreFilesystemProfile = stateStoreProperties.getOrDefault(StateStoreConstants.HAZELCAST_DISCOVERY_TCPIP_PROFILE, filesystemProfile);
                fileSystemClient = fileSystemClientManager.getFileSystemClient(stateStoreFilesystemProfile, Paths.get("/"));
            }
            LOG.info("-- Loading seed store hazelcast --");
            try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(seedStoreFactory.getClass().getClassLoader())) {
                seedStoreHazelcast = seedStoreFactory.create(clusterName, SeedStoreSubType.HAZELCAST, fileSystemClient, ImmutableMap.copyOf(config));
            }
            try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(seedStoreHazelcast.getClass().getClassLoader())) {
                // start seed refresher
                seedRefreshExecutor.scheduleWithFixedDelay(() -> refreshSeeds(SeedStoreSubType.HAZELCAST), 0, seedHeartBeat, TimeUnit.MILLISECONDS);
            }
        }
    }
}

Example 3 with SeedStoreFactory

use of io.prestosql.spi.seedstore.SeedStoreFactory in project hetu-core by openlookeng.

the class TestSeedStoreManager method setUp.

@BeforeMethod
private void setUp() throws IOException {
    FileSystemClientManager mockFileSystemClientManager = mock(FileSystemClientManager.class);
    when(mockFileSystemClientManager.getFileSystemClient(anyString(), any())).thenReturn(null);
    seedStoreManager = new SeedStoreManager(mockFileSystemClientManager);
    SeedStore mockSeedStore = new MockSeedStore();
    mockSeedStore.add(new HashSet<>());
    SeedStoreFactory mockSeedStoreFactory = mock(SeedStoreFactory.class);
    when(mockSeedStoreFactory.getName()).thenReturn("filebased");
    when(mockSeedStoreFactory.create(any(String.class), any(SeedStoreSubType.class), any(HetuFileSystemClient.class), any(Map.class))).thenReturn(mockSeedStore);
    seedStoreManager.addSeedStoreFactory(mockSeedStoreFactory);
}

Example 4 with SeedStoreFactory

use of io.prestosql.spi.seedstore.SeedStoreFactory in project hetu-core by openlookeng.

the class TestSeedStoreManager method testDupAddFactory.

@Test(expectedExceptions = IllegalArgumentException.class)
void testDupAddFactory() {
    SeedStoreFactory mockSeedStoreFactory2 = mock(SeedStoreFactory.class);
    when(mockSeedStoreFactory2.getName()).thenReturn("filebased");
    seedStoreManager.addSeedStoreFactory(mockSeedStoreFactory2);
}

Example 5 with SeedStoreFactory

use of io.prestosql.spi.seedstore.SeedStoreFactory in project hetu-core by openlookeng.

the class TestHazelcastStateStoreFactory method testUnableGetSeeds.

/**
 * Test what happens to hazelcast factory if no seeds get from seed store
 *
 * @throws IOException IOException thrown if seedStore read write errors happen
 */
@Test(expectedExceptions = RuntimeException.class)
public void testUnableGetSeeds() throws IOException {
    HazelcastStateStoreFactory factory = new HazelcastStateStoreFactory();
    assertEquals(factory.getName(), HAZELCAST);
    SeedStoreFactory seedStoreFactory = new FileBasedSeedStoreFactory();
    assertEquals(seedStoreFactory.getName(), "filebased");
    Map<String, String> properties = new HashMap<>(0);
    properties.put(STATE_STORE_CLUSTER_CONFIG_NAME, TEST_CLUSTER_NAME);
    properties.put(DISCOVERY_MODE_CONFIG_NAME, DISCOVERY_MODE_TCPIP);
    SeedStore seedStore = mock(SeedStore.class);
    when(seedStore.get()).thenReturn(ImmutableList.of());
    factory.create(TEST_STATE_STORE_NAME, seedStore, properties);
}