Examples with Files java.nio.file.Files used on opensource projects

Example 71 with Files

use of java.nio.file.Files in project infoarchive-sip-sdk by Enterprise-Content-Management.

the class FileArchiver method run.

private void run(String rootPath, String sip) throws IOException {
    File sipFile = new File(sip).getCanonicalFile();
    if (!sipFile.getParentFile().mkdirs() && !sipFile.getParentFile().isDirectory()) {
        throw new IllegalStateException("Could not create all required directories in path " + sipFile.getParentFile().getAbsolutePath());
    }
    System.out.printf("%nSample 2: Archiving files from %s into %s%n", rootPath, sipFile.getPath());
    // Tell InfoArchive where and how to archive the data
    URI entityUri = URI.create("urn:com.opentext.ia.sdk.sample.file:1.0");
    String entityName = "file";
    PackagingInformation prototype = PackagingInformation.builder().dss().application("fileApplication").holding("fileHolding").producer("SIP SDK").entity(entityName).schema(entityUri.toString()).end().build();
    // Define a mapping from our domain object to the PDI XML
    PdiAssembler<File> pdiAssembler = new XmlPdiAssembler<File>(entityUri, entityName) {

        @Override
        protected void doAdd(File file, Map<String, ContentInfo> contentInfo) {
            try {
                String path = relativePath(file, rootPath);
                getBuilder().element("path", path).element("size", Long.toString(file.length())).element("permissions", permissionsOf(file)).element("contentType", Files.probeContentType(file.toPath())).elements("hashes", "hash", contentInfo.get(path).getContentHashes(), (hash, builder) -> {
                    builder.attribute("algorithm", hash.getHashFunction()).attribute("encoding", hash.getEncoding()).attribute("value", hash.getValue());
                });
            } catch (IOException e) {
                throw new RuntimeIoException(e);
            }
        }
    };
    DigitalObjectsExtraction<File> contentAssembler = file -> Collections.singleton(DigitalObject.fromFile(relativePath(file, rootPath), file)).iterator();
    HashAssembler contentHashAssembler = new SingleHashAssembler(HashFunction.SHA256, Encoding.BASE64);
    // Assemble the SIP
    SipAssembler<File> assembler = SipAssembler.forPdiAndContentWithContentHashing(prototype, pdiAssembler, contentAssembler, contentHashAssembler);
    assembler.start(new FileBuffer(sipFile));
    try {
        addFilesIn(new File(rootPath), rootPath, relativePath(sipFile, rootPath), assembler);
    } finally {
        assembler.end();
    }
    // Show metrics about the assembly process
    SipMetrics metrics = assembler.getMetrics();
    System.out.printf("  Added %d files to SIP of %d bytes in %d ms%n", metrics.numDigitalObjects(), metrics.sipFileSize(), metrics.assemblyTime());
}

Example 72 with Files

use of java.nio.file.Files in project jena by apache.

the class FilenameUtils method scanForDirByPattern.

/**
 * Find the files in this directory that have namebase as a prefix and
 *  are then numbered.
 *  <p>
 *  Returns a sorted list from, low to high index.
 */
public static List<Path> scanForDirByPattern(Path directory, String namebase, String nameSep) {
    Pattern pattern = Pattern.compile(Pattern.quote(namebase) + Pattern.quote(nameSep) + "[\\d]+");
    List<Path> paths = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory, namebase + nameSep + "*")) {
        for (Path entry : stream) {
            if (!pattern.matcher(entry.getFileName().toString()).matches()) {
                throw new DBOpEnvException("Invalid filename for matching: " + entry.getFileName());
            // Alternative: Skip bad trailing parts but more likely there is a naming problem.
            // LOG.warn("Invalid filename for matching: {} skipped", entry.getFileName());
            // continue;
            }
            // Follows symbolic links.
            if (!Files.isDirectory(entry))
                throw new DBOpEnvException("Not a directory: " + entry);
            paths.add(entry);
        }
    } catch (IOException ex) {
        FmtLog.warn(LOG, "Can't inspect directory: (%s, %s)", directory, namebase);
        throw new DBOpEnvException(ex);
    }
    Comparator<Path> comp = (f1, f2) -> {
        int num1 = extractIndex(f1.getFileName().toString(), namebase, nameSep);
        int num2 = extractIndex(f2.getFileName().toString(), namebase, nameSep);
        return Integer.compare(num1, num2);
    };
    paths.sort(comp);
    // indexes.sort(Long::compareTo);
    return paths;
}

Example 73 with Files

use of java.nio.file.Files in project crate by crate.

the class BlobPathITest method testDataIsStoredInGlobalBlobPath.

@Test
public void testDataIsStoredInGlobalBlobPath() throws Exception {
    launchNodeAndInitClient(configureGlobalBlobPath());
    Settings indexSettings = oneShardAndZeroReplicas();
    blobAdminClient.createBlobTable("test", indexSettings).get();
    client.put("test", "abcdefg");
    String digest = "2fb5e13419fc89246865e7a324f476ec624e8740";
    try (Stream<Path> files = Files.walk(globalBlobPath)) {
        assertThat(files.anyMatch(i -> digest.equals(i.getFileName().toString())), is(true));
    }
}

Example 74 with Files

use of java.nio.file.Files in project zeppelin by apache.

the class SparkInterpreterLauncher method buildEnvFromProperties.

@Override
public Map<String, String> buildEnvFromProperties(InterpreterLaunchContext context) throws IOException {
    Map<String, String> env = super.buildEnvFromProperties(context);
    Properties sparkProperties = new Properties();
    String spMaster = getSparkMaster();
    if (spMaster != null) {
        sparkProperties.put(SPARK_MASTER_KEY, spMaster);
    }
    for (String key : properties.stringPropertyNames()) {
        String propValue = properties.getProperty(key);
        if (RemoteInterpreterUtils.isEnvString(key) && !StringUtils.isBlank(propValue)) {
            env.put(key, propValue);
        }
        if (isSparkConf(key, propValue)) {
            // instead of into sparkProperties.
            if (Objects.equals("spark.driver.extraJavaOptions", key)) {
                env.put("SPARK_DRIVER_EXTRAJAVAOPTIONS_CONF", (String) properties.remove(key));
                continue;
            }
            sparkProperties.setProperty(key, propValue);
        }
    }
    // set spark.app.name if it is not set or empty
    if (!sparkProperties.containsKey("spark.app.name") || StringUtils.isBlank(sparkProperties.getProperty("spark.app.name"))) {
        sparkProperties.setProperty("spark.app.name", context.getInterpreterGroupId());
    }
    setupPropertiesForPySpark(sparkProperties);
    setupPropertiesForSparkR(sparkProperties);
    String condaEnvName = context.getProperties().getProperty("zeppelin.interpreter.conda.env.name");
    if (StringUtils.isNotBlank(condaEnvName)) {
        if (!isYarnCluster()) {
            throw new IOException("zeppelin.interpreter.conda.env.name only works for yarn-cluster mode");
        }
        sparkProperties.setProperty("spark.pyspark.python", condaEnvName + "/bin/python");
    }
    if (isYarnCluster()) {
        env.put("ZEPPELIN_SPARK_YARN_CLUSTER", "true");
        sparkProperties.setProperty("spark.yarn.submit.waitAppCompletion", "false");
        // Need to set `zeppelin.interpreter.forceShutdown` in interpreter properties directly
        // instead of updating sparkProperties.
        // Because `zeppelin.interpreter.forceShutdown` is initialized in RemoteInterpreterServer
        // before SparkInterpreter is created.
        context.getProperties().put("zeppelin.interpreter.forceShutdown", "false");
    } else if (zConf.isOnlyYarnCluster()) {
        throw new IOException("Only yarn-cluster mode is allowed, please set " + ZeppelinConfiguration.ConfVars.ZEPPELIN_SPARK_ONLY_YARN_CLUSTER.getVarName() + " to false if you want to use other modes.");
    }
    if (isYarnMode() && getDeployMode().equals("cluster")) {
        if (sparkProperties.containsKey("spark.files")) {
            sparkProperties.put("spark.files", sparkProperties.getProperty("spark.files") + "," + zConf.getConfDir() + "/log4j_yarn_cluster.properties");
        } else {
            sparkProperties.put("spark.files", zConf.getConfDir() + "/log4j_yarn_cluster.properties");
        }
        sparkProperties.put("spark.yarn.maxAppAttempts", "1");
    }
    String scalaVersion = null;
    try {
        scalaVersion = detectSparkScalaVersion(getEnv("SPARK_HOME"), env);
        context.getProperties().put("zeppelin.spark.scala.version", scalaVersion);
    } catch (Exception e) {
        throw new IOException("Fail to detect scala version, the reason is:" + e.getMessage());
    }
    if (isYarnMode() && getDeployMode().equals("cluster")) {
        try {
            List<String> additionalJars = new ArrayList<>();
            Path localRepoPath = Paths.get(zConf.getInterpreterLocalRepoPath(), context.getInterpreterSettingId());
            if (Files.exists(localRepoPath) && Files.isDirectory(localRepoPath)) {
                try (DirectoryStream<Path> localRepoStream = Files.newDirectoryStream(localRepoPath, Files::isRegularFile)) {
                    List<String> localRepoJars = StreamSupport.stream(localRepoStream.spliterator(), false).map(jar -> jar.toAbsolutePath().toString()).collect(Collectors.toList());
                    additionalJars.addAll(localRepoJars);
                }
            }
            Path scalaFolder = Paths.get(zConf.getZeppelinHome(), "/interpreter/spark/scala-" + scalaVersion);
            if (!scalaFolder.toFile().exists()) {
                throw new IOException("spark scala folder " + scalaFolder.toFile() + " doesn't exist");
            }
            try (DirectoryStream<Path> scalaStream = Files.newDirectoryStream(scalaFolder, Files::isRegularFile)) {
                List<String> scalaJars = StreamSupport.stream(scalaStream.spliterator(), false).map(jar -> jar.toAbsolutePath().toString()).collect(Collectors.toList());
                additionalJars.addAll(scalaJars);
            }
            // add zeppelin-interpreter-shaded
            Path interpreterFolder = Paths.get(zConf.getZeppelinHome(), "/interpreter");
            try (DirectoryStream<Path> interpreterStream = Files.newDirectoryStream(interpreterFolder, Files::isRegularFile)) {
                List<String> interpreterJars = StreamSupport.stream(interpreterStream.spliterator(), false).filter(jar -> jar.toFile().getName().startsWith("zeppelin-interpreter-shaded") && jar.toFile().getName().endsWith(".jar")).map(jar -> jar.toAbsolutePath().toString()).collect(Collectors.toList());
                if (interpreterJars.isEmpty()) {
                    throw new IOException("zeppelin-interpreter-shaded jar is not found");
                } else if (interpreterJars.size() > 1) {
                    throw new IOException("more than 1 zeppelin-interpreter-shaded jars are found: " + StringUtils.join(interpreterJars, ","));
                }
                additionalJars.addAll(interpreterJars);
            }
            if (sparkProperties.containsKey("spark.jars")) {
                sparkProperties.put("spark.jars", sparkProperties.getProperty("spark.jars") + "," + StringUtils.join(additionalJars, ","));
            } else {
                sparkProperties.put("spark.jars", StringUtils.join(additionalJars, ","));
            }
        } catch (Exception e) {
            throw new IOException("Fail to set additional jars for spark interpreter", e);
        }
    }
    StringJoiner sparkConfSJ = new StringJoiner("|");
    if (context.getOption().isUserImpersonate() && zConf.getZeppelinImpersonateSparkProxyUser()) {
        sparkConfSJ.add("--proxy-user");
        sparkConfSJ.add(context.getUserName());
        sparkProperties.remove("spark.yarn.keytab");
        sparkProperties.remove("spark.yarn.principal");
    }
    for (String name : sparkProperties.stringPropertyNames()) {
        sparkConfSJ.add("--conf");
        sparkConfSJ.add(name + "=" + sparkProperties.getProperty(name) + "");
    }
    env.put("ZEPPELIN_SPARK_CONF", sparkConfSJ.toString());
    // we also fallback to zeppelin-env.sh if it is not specified in interpreter setting.
    for (String envName : new String[] { "SPARK_HOME", "SPARK_CONF_DIR", "HADOOP_CONF_DIR" }) {
        String envValue = getEnv(envName);
        if (!StringUtils.isBlank(envValue)) {
            env.put(envName, envValue);
        }
    }
    String keytab = properties.getProperty("spark.yarn.keytab", zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_KERBEROS_KEYTAB));
    String principal = properties.getProperty("spark.yarn.principal", zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_KERBEROS_PRINCIPAL));
    if (!StringUtils.isBlank(keytab) && !StringUtils.isBlank(principal)) {
        env.put("ZEPPELIN_SERVER_KERBEROS_KEYTAB", keytab);
        env.put("ZEPPELIN_SERVER_KERBEROS_PRINCIPAL", principal);
        LOGGER.info("Run Spark under secure mode with keytab: {}, principal: {}", keytab, principal);
    } else {
        LOGGER.info("Run Spark under non-secure mode as no keytab and principal is specified");
    }
    env.put("PYSPARK_PIN_THREAD", "true");
    // ZEPPELIN_INTP_CLASSPATH
    String sparkConfDir = getEnv("SPARK_CONF_DIR");
    if (StringUtils.isBlank(sparkConfDir)) {
        String sparkHome = getEnv("SPARK_HOME");
        sparkConfDir = sparkHome + "/conf";
    }
    Properties sparkDefaultProperties = new Properties();
    File sparkDefaultFile = new File(sparkConfDir, "spark-defaults.conf");
    if (sparkDefaultFile.exists()) {
        sparkDefaultProperties.load(new FileInputStream(sparkDefaultFile));
        String driverExtraClassPath = sparkDefaultProperties.getProperty("spark.driver.extraClassPath");
        if (!StringUtils.isBlank(driverExtraClassPath)) {
            env.put("ZEPPELIN_INTP_CLASSPATH", driverExtraClassPath);
        }
    } else {
        LOGGER.warn("spark-defaults.conf doesn't exist: {}", sparkDefaultFile.getAbsolutePath());
    }
    if (isYarnMode()) {
        boolean runAsLoginUser = Boolean.parseBoolean(context.getProperties().getProperty("zeppelin.spark.run.asLoginUser", "true"));
        String userName = context.getUserName();
        if (runAsLoginUser && !"anonymous".equals(userName)) {
            env.put("HADOOP_USER_NAME", userName);
        }
    }
    LOGGER.info("buildEnvFromProperties: {}", env);
    return env;
}

Example 75 with Files

use of java.nio.file.Files in project gocd by gocd.

the class JarUtilTest method shouldExtractJars.

@Test
public void shouldExtractJars() throws Exception {
    File sourceFile = new File(PATH_WITH_HASHES + "test-agent.jar");
    Set<File> files = new HashSet<>(JarUtil.extractFilesInLibDirAndReturnFiles(sourceFile, jarEntry -> jarEntry.getName().endsWith(".class"), temporaryFolder));
    Set<File> actualFiles = Files.list(temporaryFolder.toPath()).map(Path::toFile).collect(Collectors.toSet());
    assertEquals(files, actualFiles);
    assertEquals(files.size(), 2);
    Set<String> fileNames = files.stream().map(File::getName).collect(Collectors.toSet());
    assertEquals(fileNames, new HashSet<>(Arrays.asList("ArgPrintingMain.class", "HelloWorldStreamWriter.class")));
}