Examples with AlluxioConfiguration alluxio.conf.AlluxioConfiguration used on opensource projects

Example 26 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class AgentProcessMonitor method main.

/**
 * Starts the Hub Agent monitor.
 *
 * @param args command line arguments, should be empty
 */
public static void main(String[] args) {
    AlluxioConfiguration conf = ServerConfiguration.global();
    RetryPolicy policy = new ExponentialBackoffRetry(50, 1000, // Max time ~2 min
    130);
    InetSocketAddress addr = AgentRpcServer.getConfiguredAddress(conf);
    try {
        pingService(addr, policy, 5000);
    } catch (Exception e) {
        LOG.error("Failed to connect to Hub Agent RPC server @" + addr);
        System.exit(1);
    }
    System.exit(0);
}

Example 27 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class AgentRpcServerTest method testSimpleStartup.

@Test
public void testSimpleStartup() throws Exception {
    AlluxioConfiguration conf = getTestSpecificTestConfig();
    AgentRpcServer server = new AgentRpcServer(conf, TestAgentProcessContextFactory.simpleContext(conf));
    // Server is running if assigned port is != 0
    assertTrue(server.isServing());
    assertTrue(server.getPort() != 0);
}

Example 28 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class JobUtils method loadThroughRead.

private static void loadThroughRead(URIStatus status, FileSystemContext context, long blockId, AlluxioConfiguration conf) throws IOException {
    // This does not work for remote worker unless we have passive cache on.
    AlluxioProperties prop = context.getClusterConf().copyProperties();
    prop.set(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED, true);
    AlluxioConfiguration config = new InstancedConfiguration(prop);
    FileSystemContext loadContext = FileSystemContext.create(config);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(loadContext);
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo info = Preconditions.checkNotNull(status.getBlockInfo(blockId));
    try (InputStream inputStream = blockStore.getInStream(info, inOptions, ImmutableMap.of())) {
        while (inputStream.read(sIgnoredReadBuf) != -1) {
        }
    }
}

Example 29 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class JobUtils method loadBlock.

/**
 * Loads a block into the local worker. If the block doesn't exist in Alluxio, it will be read
 * from the UFS.
 * @param status the uriStatus
 * @param context filesystem context
 * @param blockId the id of the block to load
 * @param address specify a worker to load into
 * @param directCache Use passive-cache or direct cache request
 */
public static void loadBlock(URIStatus status, FileSystemContext context, long blockId, WorkerNetAddress address, boolean directCache) throws AlluxioException, IOException {
    AlluxioConfiguration conf = ServerConfiguration.global();
    // Explicitly specified a worker to load
    WorkerNetAddress localNetAddress = address;
    String localHostName = NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, conf);
    List<WorkerNetAddress> netAddress = context.getCachedWorkers().stream().map(BlockWorkerInfo::getNetAddress).filter(x -> Objects.equals(x.getHost(), localHostName)).collect(Collectors.toList());
    if (localNetAddress == null && !netAddress.isEmpty()) {
        localNetAddress = netAddress.get(0);
    }
    if (localNetAddress == null) {
        throw new NotFoundException(ExceptionMessage.NO_LOCAL_BLOCK_WORKER_LOAD_TASK.getMessage(blockId));
    }
    Set<String> pinnedLocation = status.getPinnedMediumTypes();
    if (pinnedLocation.size() > 1) {
        throw new AlluxioException(ExceptionMessage.PINNED_TO_MULTIPLE_MEDIUMTYPES.getMessage(status.getPath()));
    }
    // Only use this read local first method to load if nearest worker is clear
    if (netAddress.size() <= 1 && pinnedLocation.isEmpty() && status.isPersisted()) {
        if (directCache) {
            loadThroughCacheRequest(status, context, blockId, conf, localNetAddress);
        } else {
            loadThroughRead(status, context, blockId, conf);
        }
        return;
    }
    // TODO(bin): remove the following case when we consolidate tier and medium
    // since there is only one element in the set, we take the first element in the set
    String medium = pinnedLocation.isEmpty() ? "" : pinnedLocation.iterator().next();
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    // Set read location policy always to local first for loading blocks for job tasks
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    OutStreamOptions outOptions = OutStreamOptions.defaults(context.getClientContext());
    outOptions.setMediumType(medium);
    // Set write location policy always to local first for loading blocks for job tasks
    outOptions.setLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo blockInfo = status.getBlockInfo(blockId);
    Preconditions.checkNotNull(blockInfo, "Can not find block %s in status %s", blockId, status);
    long blockSize = blockInfo.getLength();
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context);
    try (OutputStream outputStream = blockStore.getOutStream(blockId, blockSize, localNetAddress, outOptions)) {
        try (InputStream inputStream = blockStore.getInStream(blockId, inOptions)) {
            ByteStreams.copy(inputStream, outputStream);
        } catch (Throwable t) {
            try {
                ((Cancelable) outputStream).cancel();
            } catch (Throwable t2) {
                t.addSuppressed(t2);
            }
            throw t;
        }
    }
}

Example 30 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class DistributedLoadCommand method run.

@Override
public int run(CommandLine cl) throws AlluxioException, IOException {
    mActiveJobs = FileSystemShellUtils.getIntArg(cl, ACTIVE_JOB_COUNT_OPTION, AbstractDistributedJobCommand.DEFAULT_ACTIVE_JOBS);
    System.out.format("Allow up to %s active jobs%n", mActiveJobs);
    String[] args = cl.getArgs();
    AlluxioConfiguration conf = mFsContext.getClusterConf();
    int defaultBatchSize = conf.getInt(PropertyKey.JOB_REQUEST_BATCH_SIZE);
    int replication = FileSystemShellUtils.getIntArg(cl, REPLICATION_OPTION, DEFAULT_REPLICATION);
    int batchSize = FileSystemShellUtils.getIntArg(cl, BATCH_SIZE_OPTION, defaultBatchSize);
    boolean directCache = !cl.hasOption(PASSIVE_CACHE_OPTION.getLongOpt());
    Set<String> workerSet = new HashSet<>();
    Set<String> excludedWorkerSet = new HashSet<>();
    Set<String> localityIds = new HashSet<>();
    Set<String> excludedLocalityIds = new HashSet<>();
    if (cl.hasOption(HOST_FILE_OPTION.getLongOpt())) {
        String hostFile = cl.getOptionValue(HOST_FILE_OPTION.getLongOpt()).trim();
        readLinesToSet(workerSet, hostFile);
    } else if (cl.hasOption(HOSTS_OPTION.getLongOpt())) {
        String argOption = cl.getOptionValue(HOSTS_OPTION.getLongOpt()).trim();
        readItemsFromOptionString(workerSet, argOption);
    }
    if (cl.hasOption(EXCLUDED_HOST_FILE_OPTION.getLongOpt())) {
        String hostFile = cl.getOptionValue(EXCLUDED_HOST_FILE_OPTION.getLongOpt()).trim();
        readLinesToSet(excludedWorkerSet, hostFile);
    } else if (cl.hasOption(EXCLUDED_HOSTS_OPTION.getLongOpt())) {
        String argOption = cl.getOptionValue(EXCLUDED_HOSTS_OPTION.getLongOpt()).trim();
        readItemsFromOptionString(excludedWorkerSet, argOption);
    }
    if (cl.hasOption(LOCALITY_FILE_OPTION.getLongOpt())) {
        String localityFile = cl.getOptionValue(LOCALITY_FILE_OPTION.getLongOpt()).trim();
        readLinesToSet(localityIds, localityFile);
    } else if (cl.hasOption(LOCALITY_OPTION.getLongOpt())) {
        String argOption = cl.getOptionValue(LOCALITY_OPTION.getLongOpt()).trim();
        readItemsFromOptionString(localityIds, argOption);
    }
    if (cl.hasOption(EXCLUDED_LOCALITY_FILE_OPTION.getLongOpt())) {
        String localityFile = cl.getOptionValue(EXCLUDED_LOCALITY_FILE_OPTION.getLongOpt()).trim();
        readLinesToSet(excludedLocalityIds, localityFile);
    } else if (cl.hasOption(EXCLUDED_LOCALITY_OPTION.getLongOpt())) {
        String argOption = cl.getOptionValue(EXCLUDED_LOCALITY_OPTION.getLongOpt()).trim();
        readItemsFromOptionString(excludedLocalityIds, argOption);
    }
    List<URIStatus> filePool = new ArrayList<>(batchSize);
    if (!cl.hasOption(INDEX_FILE.getLongOpt())) {
        AlluxioURI path = new AlluxioURI(args[0]);
        DistributedLoadUtils.distributedLoad(this, filePool, batchSize, path, replication, workerSet, excludedWorkerSet, localityIds, excludedLocalityIds, directCache, true);
    } else {
        try (BufferedReader reader = new BufferedReader(new FileReader(args[0]))) {
            for (String filename; (filename = reader.readLine()) != null; ) {
                AlluxioURI path = new AlluxioURI(filename);
                DistributedLoadUtils.distributedLoad(this, filePool, batchSize, path, replication, workerSet, excludedWorkerSet, localityIds, excludedLocalityIds, directCache, true);
            }
        }
    }
    System.out.println(String.format("Completed count is %d,Failed count is %d.", getCompletedCount(), getFailedCount()));
    return 0;
}