Examples with FileSystemClientManager io.prestosql.filesystem.FileSystemClientManager used on opensource projects

Example 1 with FileSystemClientManager

use of io.prestosql.filesystem.FileSystemClientManager in project hetu-core by openlookeng.

the class TestStateStoreLauncherAndProvider method testRegisterDiscoveryService.

@Test(timeOut = 5000, expectedExceptions = ThreadTimeoutException.class)
public void testRegisterDiscoveryService() throws Exception {
    String failurehost = "failurehost";
    String otherhost = "otherhost";
    String localHostName = "localhost";
    int port = 8888;
    URI uri = new URI("http://" + localHostName + ":" + port);
    MockStateMap discoveryServiceMap = new MockStateMap(DISCOVERY_SERVICE, new HashMap<>());
    // Mock
    StateStore stateStore = mock(StateStore.class);
    Lock lock = mock(ReentrantLock.class);
    InternalCommunicationConfig internalCommunicationConfig = mock(InternalCommunicationConfig.class);
    HttpServerInfo httpServerInfo = mock(HttpServerInfo.class);
    when(httpServerInfo.getHttpUri()).thenReturn(uri);
    when(internalCommunicationConfig.isHttpsRequired()).thenReturn(false);
    when(stateStore.getStateCollection(DISCOVERY_SERVICE)).thenReturn(discoveryServiceMap);
    when(stateStore.getLock(DISCOVERY_SERVICE_LOCK)).thenReturn(lock);
    EmbeddedStateStoreLauncher launcher = new EmbeddedStateStoreLauncher(new SeedStoreManager(new FileSystemClientManager()), internalCommunicationConfig, httpServerInfo, new HetuConfig());
    launcher.setStateStore(stateStore);
    when(lock.tryLock(DISCOVERY_REGISTRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)).thenReturn(true);
    // discoveryServiceMap is empty, so the current coordinator can get the lock and register itself(register=true)
    discoveryServiceMap.clear();
    assertTrue(launcher.registerDiscoveryService(failurehost));
    assertEquals(discoveryServiceMap.size(), 1);
    assertTrue(discoveryServiceMap.getAll().keySet().contains(localHostName));
    // discoveryServiceMap contains the failure host, so the current coordinator can get the lock and register itself(register=true)
    discoveryServiceMap.clear();
    discoveryServiceMap.put(failurehost, String.valueOf(port));
    assertTrue(launcher.registerDiscoveryService(failurehost));
    assertEquals(discoveryServiceMap.size(), 1);
    assertTrue(discoveryServiceMap.getAll().keySet().contains(localHostName));
    // discoveryServiceMap is already updated by other coordinator(otherhosts)
    // the current coordinator can grab the lock but will not register itself(register=false)
    discoveryServiceMap.clear();
    discoveryServiceMap.put(otherhost, String.valueOf(port));
    assertFalse(launcher.registerDiscoveryService(failurehost));
    assertEquals(discoveryServiceMap.size(), 1);
    assertFalse(discoveryServiceMap.containsKey(localHostName));
    when(lock.tryLock(DISCOVERY_REGISTRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)).thenReturn(false);
    // discoveryServiceMap is already updated by other coordinator(otherhosts)
    // the current coordinator cannot grab the lock and not register itself
    discoveryServiceMap.clear();
    discoveryServiceMap.put(otherhost, String.valueOf(port));
    assertFalse(launcher.registerDiscoveryService(failurehost));
    assertEquals(discoveryServiceMap.size(), 1);
    assertFalse(discoveryServiceMap.containsKey(localHostName));
    // discoveryServiceMap contains failure host.
    // The current coordinator cannot get the lock and retry will cause timeout exception
    discoveryServiceMap.clear();
    discoveryServiceMap.put(failurehost, String.valueOf(port));
    launcher.registerDiscoveryService(failurehost);
}

Example 2 with FileSystemClientManager

use of io.prestosql.filesystem.FileSystemClientManager in project hetu-core by openlookeng.

the class PrestoServer method run.

@Override
public void run() {
    verifyJvmRequirements();
    verifySystemTimeIsReasonable();
    Logger log = Logger.get(PrestoServer.class);
    ImmutableList.Builder<Module> modules = ImmutableList.builder();
    modules.add(new NodeModule(), Modules.override(new DiscoveryModule()).with(new HetuDiscoveryModule()), Modules.override(new HttpServerModule()).with(new HetuHttpServerModule()), new JsonModule(), new SmileModule(), new JaxrsModule(), new MBeanModule(), new PrefixObjectNameGeneratorModule("io.prestosql"), new HetuJmxModule(), new JmxHttpModule(), new LogJmxModule(), new TraceTokenModule(), new JsonEventModule(), new HttpEventModule(), new ServerSecurityModule(), new AccessControlModule(), new PasswordSecurityModule(), new EventListenerModule(), new ServerMainModule(sqlParserOptions), new NodeStateChangeModule(), new WarningCollectorModule());
    modules.addAll(getAdditionalModules());
    Bootstrap app = new Bootstrap(modules.build());
    try {
        Injector injector = app.strictConfig().initialize();
        logLocation(log, "Working directory", Paths.get("."));
        logLocation(log, "Etc directory", Paths.get("etc"));
        injector.getInstance(PluginManager.class).loadPlugins();
        FileSystemClientManager fileSystemClientManager = injector.getInstance(FileSystemClientManager.class);
        fileSystemClientManager.loadFactoryConfigs();
        injector.getInstance(SeedStoreManager.class).loadSeedStore();
        if (injector.getInstance(SeedStoreManager.class).isSeedStoreOnYarnEnabled()) {
            addSeedOnYarnInformation(injector.getInstance(ServerConfig.class), injector.getInstance(SeedStoreManager.class), (HetuHttpServerInfo) injector.getInstance(HttpServerInfo.class));
        }
        launchEmbeddedStateStore(injector.getInstance(HetuConfig.class), injector.getInstance(StateStoreLauncher.class));
        injector.getInstance(StateStoreProvider.class).loadStateStore();
        // relies on state-store
        injector.getInstance(HetuMetaStoreManager.class).loadHetuMetastore(fileSystemClientManager);
        // relies on metastore
        injector.getInstance(HeuristicIndexerManager.class).buildIndexClient();
        injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers();
        injector.getInstance(StaticCatalogStore.class).loadCatalogs();
        injector.getInstance(DynamicCatalogStore.class).loadCatalogStores(fileSystemClientManager);
        injector.getInstance(DynamicCatalogScanner.class).start();
        injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager();
        injector.getInstance(ResourceGroupManager.class).loadConfigurationManager();
        injector.getInstance(AccessControlManager.class).loadSystemAccessControl();
        injector.getInstance(PasswordAuthenticatorManager.class).loadPasswordAuthenticator();
        injector.getInstance(EventListenerManager.class).loadConfiguredEventListener();
        injector.getInstance(GroupProviderManager.class).loadConfiguredGroupProvider();
        // preload index (on coordinator only)
        if (injector.getInstance(ServerConfig.class).isCoordinator()) {
            HeuristicIndexerManager heuristicIndexerManager = injector.getInstance(HeuristicIndexerManager.class);
            heuristicIndexerManager.preloadIndex();
            heuristicIndexerManager.initCache();
        }
        // register dynamic filter listener
        registerStateStoreListeners(injector.getInstance(StateStoreListenerManager.class), injector.getInstance(DynamicFilterCacheManager.class), injector.getInstance(ServerConfig.class), injector.getInstance(NodeSchedulerConfig.class));
        // Initialize snapshot Manager
        injector.getInstance(SnapshotUtils.class).initialize();
        injector.getInstance(Announcer.class).start();
        injector.getInstance(ServerInfoResource.class).startupComplete();
        log.info("======== SERVER STARTED ========");
    } catch (Throwable e) {
        log.error(e);
        System.exit(1);
    }
}

Example 3 with FileSystemClientManager

use of io.prestosql.filesystem.FileSystemClientManager in project hetu-core by openlookeng.

the class TestSeedStoreManager method setUp.

@BeforeMethod
private void setUp() throws IOException {
    FileSystemClientManager mockFileSystemClientManager = mock(FileSystemClientManager.class);
    when(mockFileSystemClientManager.getFileSystemClient(anyString(), any())).thenReturn(null);
    seedStoreManager = new SeedStoreManager(mockFileSystemClientManager);
    SeedStore mockSeedStore = new MockSeedStore();
    mockSeedStore.add(new HashSet<>());
    SeedStoreFactory mockSeedStoreFactory = mock(SeedStoreFactory.class);
    when(mockSeedStoreFactory.getName()).thenReturn("filebased");
    when(mockSeedStoreFactory.create(any(String.class), any(SeedStoreSubType.class), any(HetuFileSystemClient.class), any(Map.class))).thenReturn(mockSeedStore);
    seedStoreManager.addSeedStoreFactory(mockSeedStoreFactory);
}

Example 4 with FileSystemClientManager

use of io.prestosql.filesystem.FileSystemClientManager in project hetu-core by openlookeng.

the class TestTaskSnapshotManager method setup.

@BeforeMethod
public void setup() throws Exception {
    // Set up snapshot config
    snapshotConfig = new SnapshotConfig();
    // Set up mock file system client manager
    fileSystemClientManager = mock(FileSystemClientManager.class);
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(SNAPSHOT_FILE_SYSTEM_DIR)));
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.rootPath = SNAPSHOT_FILE_SYSTEM_DIR;
    snapshotUtils.initialize();
}

Example 5 with FileSystemClientManager

use of io.prestosql.filesystem.FileSystemClientManager in project hetu-core by openlookeng.

the class TestBinaryFileSpiller method setUp.

@BeforeMethod
public void setUp() throws IOException {
    Metadata metadata = createTestMetadataManager();
    FileSystemClientManager fileSystemClientManager = mock(FileSystemClientManager.class);
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
    spillerStats = new SpillerStats();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    try {
        featuresConfig.setSpillerSpillPaths(spillPath.getCanonicalPath());
    } catch (IOException e) {
        System.out.println(e.getStackTrace());
    }
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
    singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(metadata, spillerStats, featuresConfig, nodeSpillConfig, fileSystemClientManager);
    factory = new GenericSpillerFactory(singleStreamSpillerFactory);
    PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), nodeSpillConfig.isSpillCompressionEnabled());
    pagesSerde = pagesSerdeFactory.createPagesSerde();
    memoryContext = newSimpleAggregatedMemoryContext();
}