Examples with Time org.apache.flink.api.common.time.Time used on opensource projects

Example 1 with Time

use of org.apache.flink.api.common.time.Time in project flink by apache.

the class StackTraceSampleCoordinatorTest method testTriggerStackTraceSample.

/** Tests simple trigger and collect of stack trace samples. */
@Test
public void testTriggerStackTraceSample() throws Exception {
    ExecutionVertex[] vertices = new ExecutionVertex[] { mockExecutionVertex(new ExecutionAttemptID(), ExecutionState.RUNNING, true), mockExecutionVertex(new ExecutionAttemptID(), ExecutionState.RUNNING, true), mockExecutionVertex(new ExecutionAttemptID(), ExecutionState.RUNNING, true), mockExecutionVertex(new ExecutionAttemptID(), ExecutionState.RUNNING, true) };
    int numSamples = 1;
    Time delayBetweenSamples = Time.milliseconds(100L);
    int maxStackTraceDepth = 0;
    Future<StackTraceSample> sampleFuture = coord.triggerStackTraceSample(vertices, numSamples, delayBetweenSamples, maxStackTraceDepth);
    // Verify messages have been sent
    for (ExecutionVertex vertex : vertices) {
        ExecutionAttemptID expectedExecutionId = vertex.getCurrentExecutionAttempt().getAttemptId();
        TriggerStackTraceSample expectedMsg = new TriggerStackTraceSample(0, expectedExecutionId, numSamples, delayBetweenSamples, maxStackTraceDepth);
        verify(vertex.getCurrentExecutionAttempt()).requestStackTraceSample(eq(0), eq(numSamples), eq(delayBetweenSamples), eq(maxStackTraceDepth), any(Time.class));
    }
    assertFalse(sampleFuture.isDone());
    StackTraceElement[] stackTraceSample = Thread.currentThread().getStackTrace();
    List<StackTraceElement[]> traces = new ArrayList<>();
    traces.add(stackTraceSample);
    traces.add(stackTraceSample);
    traces.add(stackTraceSample);
    // Collect stack traces
    for (int i = 0; i < vertices.length; i++) {
        ExecutionAttemptID executionId = vertices[i].getCurrentExecutionAttempt().getAttemptId();
        coord.collectStackTraces(0, executionId, traces);
        if (i == vertices.length - 1) {
            assertTrue(sampleFuture.isDone());
        } else {
            assertFalse(sampleFuture.isDone());
        }
    }
    // Verify completed stack trace sample
    StackTraceSample sample = sampleFuture.get();
    assertEquals(0, sample.getSampleId());
    assertTrue(sample.getEndTime() >= sample.getStartTime());
    Map<ExecutionAttemptID, List<StackTraceElement[]>> tracesByTask = sample.getStackTraces();
    for (ExecutionVertex vertex : vertices) {
        ExecutionAttemptID executionId = vertex.getCurrentExecutionAttempt().getAttemptId();
        List<StackTraceElement[]> sampleTraces = tracesByTask.get(executionId);
        assertNotNull("Task not found", sampleTraces);
        assertTrue(traces.equals(sampleTraces));
    }
    // Verify no more pending sample
    assertEquals(0, coord.getNumberOfPendingSamples());
    // Verify no error on late collect
    coord.collectStackTraces(0, vertices[0].getCurrentExecutionAttempt().getAttemptId(), traces);
}

Example 2 with Time

use of org.apache.flink.api.common.time.Time in project flink by apache.

the class TaskManagerConfiguration method fromConfiguration.

// --------------------------------------------------------------------------------------------
//  Static factory methods
// --------------------------------------------------------------------------------------------
public static TaskManagerConfiguration fromConfiguration(Configuration configuration) {
    int numberSlots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
    if (numberSlots == -1) {
        numberSlots = 1;
    }
    final String[] tmpDirPaths = configuration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
    final Time timeout;
    try {
        timeout = Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis());
    } catch (Exception e) {
        throw new IllegalArgumentException("Invalid format for '" + ConfigConstants.AKKA_ASK_TIMEOUT + "'.Use formats like '50 s' or '1 min' to specify the timeout.");
    }
    LOG.info("Messages have a max timeout of " + timeout);
    final long cleanupInterval = configuration.getLong(ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL, ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
    final Time finiteRegistrationDuration;
    try {
        Duration maxRegistrationDuration = Duration.create(configuration.getString(ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION));
        if (maxRegistrationDuration.isFinite()) {
            finiteRegistrationDuration = Time.milliseconds(maxRegistrationDuration.toMillis());
        } else {
            finiteRegistrationDuration = null;
        }
    } catch (NumberFormatException e) {
        throw new IllegalArgumentException("Invalid format for parameter " + ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, e);
    }
    final Time initialRegistrationPause;
    try {
        Duration pause = Duration.create(configuration.getString(ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE));
        if (pause.isFinite()) {
            initialRegistrationPause = Time.milliseconds(pause.toMillis());
        } else {
            throw new IllegalArgumentException("The initial registration pause must be finite: " + pause);
        }
    } catch (NumberFormatException e) {
        throw new IllegalArgumentException("Invalid format for parameter " + ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
    }
    final Time maxRegistrationPause;
    try {
        Duration pause = Duration.create(configuration.getString(ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE, ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE));
        if (pause.isFinite()) {
            maxRegistrationPause = Time.milliseconds(pause.toMillis());
        } else {
            throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause);
        }
    } catch (NumberFormatException e) {
        throw new IllegalArgumentException("Invalid format for parameter " + ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
    }
    final Time refusedRegistrationPause;
    try {
        Duration pause = Duration.create(configuration.getString(ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE, ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE));
        if (pause.isFinite()) {
            refusedRegistrationPause = Time.milliseconds(pause.toMillis());
        } else {
            throw new IllegalArgumentException("The refused registration pause must be finite: " + pause);
        }
    } catch (NumberFormatException e) {
        throw new IllegalArgumentException("Invalid format for parameter " + ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
    }
    final boolean exitOnOom = configuration.getBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY);
    return new TaskManagerConfiguration(numberSlots, tmpDirPaths, timeout, finiteRegistrationDuration, initialRegistrationPause, maxRegistrationPause, refusedRegistrationPause, cleanupInterval, configuration, exitOnOom);
}

Example 3 with Time

use of org.apache.flink.api.common.time.Time in project flink by apache.

the class TaskManagerRunner method createRpcService.

// --------------------------------------------------------------------------------------------
//  Static utilities
// --------------------------------------------------------------------------------------------
/**
	 * Create a RPC service for the task manager.
	 *
	 * @param configuration The configuration for the TaskManager.
	 * @param haServices to use for the task manager hostname retrieval
	 */
public static RpcService createRpcService(final Configuration configuration, final HighAvailabilityServices haServices) throws Exception {
    checkNotNull(configuration);
    checkNotNull(haServices);
    String taskManagerHostname = configuration.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, null);
    if (taskManagerHostname != null) {
        LOG.info("Using configured hostname/address for TaskManager: {}.", taskManagerHostname);
    } else {
        Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis());
        InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(haServices.getResourceManagerLeaderRetriever(), lookupTimeout);
        taskManagerHostname = taskManagerAddress.getHostName();
        LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.", taskManagerHostname, taskManagerAddress.getHostAddress());
    }
    final int rpcPort = configuration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 0);
    Preconditions.checkState(rpcPort >= 0 && rpcPort <= 65535, "Invalid value for " + "'%s' (port for the TaskManager actor system) : %d - Leave config parameter empty or " + "use 0 to let the system choose port automatically.", ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, rpcPort);
    return RpcServiceUtils.createRpcService(taskManagerHostname, rpcPort, configuration);
}

Example 4 with Time

use of org.apache.flink.api.common.time.Time in project flink by apache.

the class AkkaInvocationHandler method invoke.

@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
    Class<?> declaringClass = method.getDeclaringClass();
    Object result;
    if (declaringClass.equals(AkkaGateway.class) || declaringClass.equals(MainThreadExecutable.class) || declaringClass.equals(Object.class) || declaringClass.equals(StartStoppable.class) || declaringClass.equals(RpcGateway.class) || declaringClass.equals(SelfGateway.class)) {
        result = method.invoke(this, args);
    } else {
        String methodName = method.getName();
        Class<?>[] parameterTypes = method.getParameterTypes();
        Annotation[][] parameterAnnotations = method.getParameterAnnotations();
        Time futureTimeout = extractRpcTimeout(parameterAnnotations, args, timeout);
        Tuple2<Class<?>[], Object[]> filteredArguments = filterArguments(parameterTypes, parameterAnnotations, args);
        RpcInvocation rpcInvocation;
        if (isLocal) {
            rpcInvocation = new LocalRpcInvocation(methodName, filteredArguments.f0, filteredArguments.f1);
        } else {
            try {
                RemoteRpcInvocation remoteRpcInvocation = new RemoteRpcInvocation(methodName, filteredArguments.f0, filteredArguments.f1);
                if (remoteRpcInvocation.getSize() > maximumFramesize) {
                    throw new IOException("The rpc invocation size exceeds the maximum akka framesize.");
                } else {
                    rpcInvocation = remoteRpcInvocation;
                }
            } catch (IOException e) {
                LOG.warn("Could not create remote rpc invocation message. Failing rpc invocation because...", e);
                throw e;
            }
        }
        Class<?> returnType = method.getReturnType();
        if (returnType.equals(Void.TYPE)) {
            rpcEndpoint.tell(rpcInvocation, ActorRef.noSender());
            result = null;
        } else if (returnType.equals(Future.class)) {
            // execute an asynchronous call
            result = new FlinkFuture<>(Patterns.ask(rpcEndpoint, rpcInvocation, futureTimeout.toMilliseconds()));
        } else {
            // execute a synchronous call
            scala.concurrent.Future<?> scalaFuture = Patterns.ask(rpcEndpoint, rpcInvocation, futureTimeout.toMilliseconds());
            Future<?> futureResult = new FlinkFuture<>(scalaFuture);
            return futureResult.get(futureTimeout.getSize(), futureTimeout.getUnit());
        }
    }
    return result;
}

Example 5 with Time

use of org.apache.flink.api.common.time.Time in project flink by apache.

the class RpcServiceUtils method createRpcService.

// ------------------------------------------------------------------------
//  RPC instantiation
// ------------------------------------------------------------------------
/**
	 * Utility method to create RPC service from configuration and hostname, port.
	 *
	 * @param hostname   The hostname/address that describes the TaskManager's data location.
	 * @param port           If true, the TaskManager will not initiate the TCP network stack.
	 * @param configuration                 The configuration for the TaskManager.
	 * @return   The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
	 * @throws IOException      Thrown, if the actor system can not bind to the address
	 * @throws Exception      Thrown is some other error occurs while creating akka actor system
	 */
public static RpcService createRpcService(String hostname, int port, Configuration configuration) throws Exception {
    LOG.info("Starting AkkaRpcService at {}.", NetUtils.hostAndPortToUrlString(hostname, port));
    final ActorSystem actorSystem;
    try {
        Config akkaConfig;
        if (hostname != null && !hostname.isEmpty()) {
            // remote akka config
            akkaConfig = AkkaUtils.getAkkaConfig(configuration, hostname, port);
        } else {
            // local akka config
            akkaConfig = AkkaUtils.getAkkaConfig(configuration);
        }
        LOG.debug("Using akka configuration \n {}.", akkaConfig);
        actorSystem = AkkaUtils.createActorSystem(akkaConfig);
    } catch (Throwable t) {
        if (t instanceof ChannelException) {
            Throwable cause = t.getCause();
            if (cause != null && t.getCause() instanceof java.net.BindException) {
                String address = NetUtils.hostAndPortToUrlString(hostname, port);
                throw new IOException("Unable to bind AkkaRpcService actor system to address " + address + " - " + cause.getMessage(), t);
            }
        }
        throw new Exception("Could not create TaskManager actor system", t);
    }
    final Time timeout = Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis());
    return new AkkaRpcService(actorSystem, timeout);
}