Examples with InstancedConfiguration alluxio.conf.InstancedConfiguration used on opensource projects

Example 41 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class MultiMount method main.

/**
 * Entry point for the {@link MultiMount} program.
 *
 * @param args command-line arguments
 */
public static void main(String[] args) {
    if (args.length != 1) {
        System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
        System.exit(-1);
    }
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3a://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.create(alluxioConf);
    try {
        // Make sure mount directory exists.
        if (!fileSystem.exists(mntPath)) {
            System.out.print("creating " + mntPath + " ... ");
            fileSystem.createDirectory(mntPath);
            System.out.println("done");
        }
        // Make sure the S3 mount point does not exist.
        if (fileSystem.exists(s3Mount)) {
            System.out.print("unmounting " + s3Mount + " ... ");
            fileSystem.unmount(s3Mount);
            System.out.println("done");
        }
        // Make sure the HDFS mount point does not exist.
        if (fileSystem.exists(hdfsMount)) {
            System.out.print("unmounting " + hdfsMount + " ... ");
            fileSystem.unmount(hdfsMount);
            System.out.println("done");
        }
        // Mount S3.
        System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
        fileSystem.mount(s3Mount, s3Path);
        System.out.println("done");
        // Mount HDFS.
        System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
        fileSystem.mount(hdfsMount, hdfsPath);
        System.out.println("done");
        // Make sure output file does not exist.
        if (fileSystem.exists(outputPath)) {
            System.out.print("deleting " + outputPath + " ... ");
            fileSystem.delete(outputPath);
            System.out.println("done");
        }
        // Open the input stream.
        System.out.print("opening " + inputPath + " ... ");
        FileInStream is = fileSystem.openFile(inputPath);
        System.out.println("done");
        // Open the output stream, setting the write type to make sure result is persisted.
        System.out.print("opening " + outputPath + " ... ");
        CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build();
        FileOutStream os = fileSystem.createFile(outputPath, options);
        System.out.println("done");
        // Copy the data
        System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
        IOUtils.copy(is, os);
        System.out.println("done");
        // Close the input stream.
        System.out.print("closing " + inputPath + " ... ");
        is.close();
        System.out.println("done");
        // Close the output stream.
        System.out.print("closing " + outputPath + " ... ");
        os.close();
        System.out.println("done");
    } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
    } finally {
        // Make sure the S3 mount point is removed.
        try {
            if (fileSystem.exists(s3Mount)) {
                System.out.print("unmounting " + s3Mount + " ... ");
                fileSystem.unmount(s3Mount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
        // Make sure the HDFS mount point is removed.
        try {
            if (fileSystem.exists(hdfsMount)) {
                System.out.print("unmounting " + hdfsMount + " ... ");
                fileSystem.unmount(hdfsMount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
    }
}

Example 42 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class AgentProcessContextTest method testHeartbeatSuccess.

@Test
public void testHeartbeatSuccess() throws Exception {
    InstancedConfiguration config = getTestSpecificConfig();
    AgentProcessContext ctx = new AgentProcessContext(config, mClient);
    CommonUtils.waitFor("cluster size to be > 0", () -> mContext.getAlluxioCluster().size() > 0, WaitForOptions.defaults().setInterval(100).setTimeoutMs(5000));
    assertEquals(1, mContext.getAlluxioCluster().size());
    AlluxioCluster c = mContext.getAlluxioCluster().toProto();
    AlluxioNodeStatus nodeStatus = c.getNode(0);
    String host = NetworkAddressUtils.getConnectHost(NetworkAddressUtils.ServiceType.HUB_AGENT_RPC, config);
    assertEquals(host, nodeStatus.getHostname());
    assertThrows(Exception.class, ctx::startHeartbeat);
}

Example 43 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class AgentProcessMonitorTest method testProcessMonitorSuccess.

@Test
public void testProcessMonitorSuccess() throws Exception {
    InstancedConfiguration c = getTestConfig();
    mTempDir.newFolder("bin");
    mTempDir.newFile("bin/alluxio-start.sh");
    c.set(PropertyKey.HOME, mTempDir.getRoot().getAbsolutePath());
    try (ManagerProcess mProc = new ManagerProcess(c)) {
        InstancedConfiguration agentConf = new InstancedConfiguration(c.copyProperties());
        agentConf.set(PropertyKey.HUB_MANAGER_RPC_PORT, mProc.getRpcPort());
        try (AgentProcess proc = new AgentProcess(agentConf)) {
            CountingRetry cntRetry = new CountingRetry(5);
            AgentProcessMonitor.pingService(new InetSocketAddress("127.0.0.1", proc.getRpcPort()), cntRetry, 200);
        }
    } catch (Exception e) {
        fail("Should not have thrown an exception: " + e);
    }
}

Example 44 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class JobUtils method loadThroughRead.

private static void loadThroughRead(URIStatus status, FileSystemContext context, long blockId, AlluxioConfiguration conf) throws IOException {
    // This does not work for remote worker unless we have passive cache on.
    AlluxioProperties prop = context.getClusterConf().copyProperties();
    prop.set(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED, true);
    AlluxioConfiguration config = new InstancedConfiguration(prop);
    FileSystemContext loadContext = FileSystemContext.create(config);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(loadContext);
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo info = Preconditions.checkNotNull(status.getBlockInfo(blockId));
    try (InputStream inputStream = blockStore.getInStream(info, inOptions, ImmutableMap.of())) {
        while (inputStream.read(sIgnoredReadBuf) != -1) {
        }
    }
}

Example 45 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class FileSystemShell method main.

/**
 * Main method, starts a new FileSystemShell.
 *
 * @param argv array of arguments given by the user's input from the terminal
 */
public static void main(String[] argv) throws IOException {
    int ret;
    InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    if (!ConfigurationUtils.masterHostConfigured(conf) && argv.length > 0 && !argv[0].equals("help")) {
        System.out.println(ConfigurationUtils.getMasterHostNotConfiguredMessage("Alluxio fs shell"));
        System.exit(1);
    }
    // Reduce the RPC retry max duration to fall earlier for CLIs
    conf.set(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "5s", Source.DEFAULT);
    try (FileSystemShell shell = new FileSystemShell(conf)) {
        ret = shell.run(argv);
    }
    System.exit(ret);
}