Examples with NodeTtlFetcherFactory com.facebook.presto.spi.ttl.NodeTtlFetcherFactory used on opensource projects

Example 1 with NodeTtlFetcherFactory

use of com.facebook.presto.spi.ttl.NodeTtlFetcherFactory in project presto by prestodb.

the class TestConfidenceBasedClusterTtlProviderManager method setup.

@BeforeClass
public void setup() {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("prism"), ImmutableSet.of(node1, node2, node3));
    ConfidenceBasedNodeTtlFetcherManager nodeTtlFetcherManager = new ConfidenceBasedNodeTtlFetcherManager(nodeManager, new NodeSchedulerConfig(), new NodeTtlFetcherManagerConfig());
    NodeTtlFetcherFactory nodeTtlFetcherFactory = new TestingNodeTtlFetcherFactory(nodeToTtl);
    nodeTtlFetcherManager.addNodeTtlFetcherFactory(nodeTtlFetcherFactory);
    nodeTtlFetcherManager.load(nodeTtlFetcherFactory.getName(), ImmutableMap.of());
    nodeTtlFetcherManager.refreshTtlInfo();
    clusterTtlProviderManager = new ConfidenceBasedClusterTtlProviderManager(nodeTtlFetcherManager);
    ClusterTtlProviderFactory clusterTtlProviderFactory = new TestingClusterTtlProviderFactory();
    clusterTtlProviderManager.addClusterTtlProviderFactory(clusterTtlProviderFactory);
    clusterTtlProviderManager.load(clusterTtlProviderFactory.getName(), ImmutableMap.of());
}

Example 2 with NodeTtlFetcherFactory

use of com.facebook.presto.spi.ttl.NodeTtlFetcherFactory in project presto by prestodb.

the class TestConfidenceBasedNodeTtlFetcherManager method setup.

@BeforeClass
public void setup() {
    nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("prism"), ImmutableSet.of(node1, node2, node3));
    nodeTtlFetcherManager = new ConfidenceBasedNodeTtlFetcherManager(nodeManager, new NodeSchedulerConfig(), new NodeTtlFetcherManagerConfig());
    NodeTtlFetcherFactory factory = new TestingNodeTtlFetcherFactory(nodeToTtl);
    nodeTtlFetcherManager.addNodeTtlFetcherFactory(factory);
    nodeTtlFetcherManager.load(factory.getName(), ImmutableMap.of());
    nodeTtlFetcherManager.refreshTtlInfo();
}

Example 3 with NodeTtlFetcherFactory

use of com.facebook.presto.spi.ttl.NodeTtlFetcherFactory in project presto by prestodb.

the class PluginManager method installPlugin.

public void installPlugin(Plugin plugin) {
    for (BlockEncoding blockEncoding : plugin.getBlockEncodings()) {
        log.info("Registering block encoding %s", blockEncoding.getName());
        blockEncodingManager.addBlockEncoding(blockEncoding);
    }
    for (Type type : plugin.getTypes()) {
        log.info("Registering type %s", type.getTypeSignature());
        metadata.getFunctionAndTypeManager().addType(type);
    }
    for (ParametricType parametricType : plugin.getParametricTypes()) {
        log.info("Registering parametric type %s", parametricType.getName());
        metadata.getFunctionAndTypeManager().addParametricType(parametricType);
    }
    for (ConnectorFactory connectorFactory : plugin.getConnectorFactories()) {
        if (disabledConnectors.contains(connectorFactory.getName())) {
            log.info("Skipping disabled connector %s", connectorFactory.getName());
            continue;
        }
        log.info("Registering connector %s", connectorFactory.getName());
        connectorManager.addConnectorFactory(connectorFactory);
    }
    for (Class<?> functionClass : plugin.getFunctions()) {
        log.info("Registering functions from %s", functionClass.getName());
        metadata.registerBuiltInFunctions(extractFunctions(functionClass));
    }
    for (FunctionNamespaceManagerFactory functionNamespaceManagerFactory : plugin.getFunctionNamespaceManagerFactories()) {
        log.info("Registering function namespace manager %s", functionNamespaceManagerFactory.getName());
        metadata.getFunctionAndTypeManager().addFunctionNamespaceFactory(functionNamespaceManagerFactory);
    }
    for (SessionPropertyConfigurationManagerFactory sessionConfigFactory : plugin.getSessionPropertyConfigurationManagerFactories()) {
        log.info("Registering session property configuration manager %s", sessionConfigFactory.getName());
        sessionPropertyDefaults.addConfigurationManagerFactory(sessionConfigFactory);
    }
    for (ResourceGroupConfigurationManagerFactory configurationManagerFactory : plugin.getResourceGroupConfigurationManagerFactories()) {
        log.info("Registering resource group configuration manager %s", configurationManagerFactory.getName());
        resourceGroupManager.addConfigurationManagerFactory(configurationManagerFactory);
    }
    for (SystemAccessControlFactory accessControlFactory : plugin.getSystemAccessControlFactories()) {
        log.info("Registering system access control %s", accessControlFactory.getName());
        accessControlManager.addSystemAccessControlFactory(accessControlFactory);
    }
    for (PasswordAuthenticatorFactory authenticatorFactory : plugin.getPasswordAuthenticatorFactories()) {
        log.info("Registering password authenticator %s", authenticatorFactory.getName());
        passwordAuthenticatorManager.addPasswordAuthenticatorFactory(authenticatorFactory);
    }
    for (EventListenerFactory eventListenerFactory : plugin.getEventListenerFactories()) {
        log.info("Registering event listener %s", eventListenerFactory.getName());
        eventListenerManager.addEventListenerFactory(eventListenerFactory);
    }
    for (TempStorageFactory tempStorageFactory : plugin.getTempStorageFactories()) {
        log.info("Registering temp storage %s", tempStorageFactory.getName());
        tempStorageManager.addTempStorageFactory(tempStorageFactory);
    }
    for (QueryPrerequisitesFactory queryPrerequisitesFactory : plugin.getQueryPrerequisitesFactories()) {
        log.info("Registering query prerequisite factory %s", queryPrerequisitesFactory.getName());
        queryPrerequisitesManager.addQueryPrerequisitesFactory(queryPrerequisitesFactory);
    }
    for (NodeTtlFetcherFactory nodeTtlFetcherFactory : plugin.getNodeTtlFetcherFactories()) {
        log.info("Registering Ttl fetcher factory %s", nodeTtlFetcherFactory.getName());
        nodeTtlFetcherManager.addNodeTtlFetcherFactory(nodeTtlFetcherFactory);
    }
    for (ClusterTtlProviderFactory clusterTtlProviderFactory : plugin.getClusterTtlProviderFactories()) {
        log.info("Registering Cluster Ttl provider factory %s", clusterTtlProviderFactory.getName());
        clusterTtlProviderManager.addClusterTtlProviderFactory(clusterTtlProviderFactory);
    }
}

Example 4 with NodeTtlFetcherFactory

use of com.facebook.presto.spi.ttl.NodeTtlFetcherFactory in project presto by prestodb.

the class ConfidenceBasedNodeTtlFetcherManager method load.

@VisibleForTesting
public void load(String factoryName, Map<String, String> properties) {
    log.info("-- Loading node ttl fetcher factory --");
    NodeTtlFetcherFactory nodeTtlFetcherFactory = ttlFetcherFactories.get(factoryName);
    checkState(nodeTtlFetcherFactory != null, "Node ttl fetcher factory %s is not registered", factoryName);
    NodeTtlFetcher nodeTtlFetcher = nodeTtlFetcherFactory.create(properties);
    checkState(this.ttlFetcher.compareAndSet(null, nodeTtlFetcher), "Node ttl fetcher has already been set!");
    log.info("-- Loaded node ttl fetcher %s --", factoryName);
}

Example 5 with NodeTtlFetcherFactory

use of com.facebook.presto.spi.ttl.NodeTtlFetcherFactory in project presto by prestodb.

the class TestNodeScheduler method testTtlAwareScheduling.

@Test
public void testTtlAwareScheduling() {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
    InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
    InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
    List<InternalNode> nodes = ImmutableList.of(node1, node2, node3);
    nodeManager.addNode(CONNECTOR_ID, nodes);
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    Instant currentInstant = Instant.now();
    NodeTtl ttl1 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(5, ChronoUnit.MINUTES).getEpochSecond(), 100)));
    NodeTtl ttl2 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(30, ChronoUnit.MINUTES).getEpochSecond(), 100)));
    NodeTtl ttl3 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(2, ChronoUnit.HOURS).getEpochSecond(), 100)));
    Map<NodeInfo, NodeTtl> nodeToTtl = ImmutableMap.of(new NodeInfo(node1.getNodeIdentifier(), node1.getHost()), ttl1, new NodeInfo(node2.getNodeIdentifier(), node2.getHost()), ttl2, new NodeInfo(node3.getNodeIdentifier(), node3.getHost()), ttl3);
    ConfidenceBasedNodeTtlFetcherManager nodeTtlFetcherManager = new ConfidenceBasedNodeTtlFetcherManager(nodeManager, new NodeSchedulerConfig(), new NodeTtlFetcherManagerConfig());
    NodeTtlFetcherFactory nodeTtlFetcherFactory = new TestingNodeTtlFetcherFactory(nodeToTtl);
    nodeTtlFetcherManager.addNodeTtlFetcherFactory(nodeTtlFetcherFactory);
    nodeTtlFetcherManager.load(nodeTtlFetcherFactory.getName(), ImmutableMap.of());
    nodeTtlFetcherManager.refreshTtlInfo();
    TestingQueryManager queryManager = new TestingQueryManager();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, nodeTtlFetcherManager, queryManager, new SimpleTtlNodeSelectorConfig());
    // Query is estimated to take 20 mins and has been executing for 3 mins, i.e, 17 mins left
    // So only node2 and node3 have enough TTL to run additional work
    Session session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(20, TimeUnit.MINUTES));
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(3, TimeUnit.MINUTES));
    assertEquals(ImmutableSet.copyOf(nodeSelector.selectRandomNodes(3)), ImmutableSet.of(node2, node3));
    // Query is estimated to take 1 hour and has been executing for 45 mins, i.e, 15 mins left
    // So only node2 and node3 have enough TTL to work on new splits
    session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(45, TimeUnit.MINUTES));
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 2; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 2);
    assertTrue(assignments.keySet().contains(node2));
    assertTrue(assignments.keySet().contains(node3));
    // Query is estimated to take 1 hour and has been executing for 20 mins, i.e, 40 mins left
    // So only node3 has enough TTL to work on new splits
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    TaskId taskId = new TaskId("test", 1, 0, 1);
    RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, node2, ImmutableList.of(), nodeTaskMap.createTaskStatsTracker(node2, taskId));
    taskMap.put(node2, newRemoteTask);
    nodeTaskMap.addTask(node2, newRemoteTask);
    session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(20, TimeUnit.MINUTES));
    splits.clear();
    for (int i = 0; i < 2; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 2);
    assertEquals(assignments.keySet().size(), 1);
    assertTrue(assignments.keySet().contains(node3));
}