Examples with ManifestJSON io.hops.hopsworks.dela.old_dto.ManifestJSON used on opensource projects

Example 1 with ManifestJSON

use of io.hops.hopsworks.dela.old_dto.ManifestJSON in project hopsworks by logicalclocks.

the class DelaHdfsController method createManifest.

private ManifestJSON createManifest(Project project, Dataset dataset, Users user) throws DelaException {
    String hdfsUser = hdfsUsersBean.getHdfsUserName(project, user);
    DistributedFileSystemOps dfso = dfs.getDfsOps(hdfsUser);
    Path datasetPath = datasetCtrl.getDatasetPath(dataset);
    ManifestJSON manifest = new ManifestJSON();
    manifest.setDatasetName(dataset.getName());
    manifest.setDatasetDescription(dataset.getDescription());
    manifest.setKafkaSupport(false);
    List<Inode> datasetFiles = new LinkedList<>();
    Map<String, Inode> avroFiles = new HashMap<>();
    for (Inode i : inodeController.getChildren(dataset.getInode())) {
        if (i.isDir()) {
            throw new DelaException(RESTCodes.DelaErrorCode.SUBDIRS_NOT_SUPPORTED, Level.FINE, DelaException.Source.LOCAL);
        }
        if (isAvro(i.getInodePK().getName())) {
            avroFiles.put(i.getInodePK().getName(), i);
        } else {
            datasetFiles.add(i);
        }
    }
    List<FileInfo> fileInfos = new LinkedList<>();
    for (Inode i : datasetFiles) {
        String fileName = i.getInodePK().getName();
        FileInfo fileInfo = new FileInfo();
        fileInfo.setFileName(fileName);
        Path filePath = new Path(datasetPath, fileName);
        try {
            fileInfo.setLength(dfso.getLength(filePath));
        } catch (IOException ex) {
            throw new DelaException(RESTCodes.DelaErrorCode.ACCESS_ERROR, Level.SEVERE, DelaException.Source.HDFS, null, ex.getMessage(), ex);
        }
        if (avroFiles.containsKey(fileName + ".avro")) {
            Path avroSchemaPath = new Path(datasetPath, filePath + ".avro");
            fileInfo.setSchema(new String(read(project, user, avroSchemaPath)));
            manifest.setKafkaSupport(true);
        } else {
            fileInfo.setSchema("");
        }
        fileInfos.add(fileInfo);
    }
    for (Inode i : avroFiles.values()) {
        String fileName = i.getInodePK().getName();
        FileInfo fileInfo = new FileInfo();
        fileInfo.setFileName(fileName);
        fileInfo.setSchema("");
        String filePath = datasetPath + File.separator + fileName;
        fileInfo.setLength(dfso.getlength(filePath));
        fileInfos.add(fileInfo);
    }
    manifest.setFileInfos(fileInfos);
    DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    manifest.setCreatorDate(dateFormat.format(new Date()));
    manifest.setCreatorEmail(user.getEmail());
    // TODO other schemas
    manifest.setMetaDataJsons(new ArrayList<>());
    return manifest;
}

Example 2 with ManifestJSON

use of io.hops.hopsworks.dela.old_dto.ManifestJSON in project hopsworks by logicalclocks.

the class DelaHdfsController method readManifest.

public ManifestJSON readManifest(Project project, Dataset dataset, Users user) throws DelaException {
    byte[] manifestBytes = read(project, user, manifestPath(dataset));
    ManifestJSON manifest = ManifestHelper.unmarshall(manifestBytes);
    return manifest;
}

Example 3 with ManifestJSON

use of io.hops.hopsworks.dela.old_dto.ManifestJSON in project hopsworks by logicalclocks.

the class DelaHdfsController method writeManifest.

public ManifestJSON writeManifest(Project project, Dataset dataset, Users user) throws DelaException {
    if (inodeController.getChildren(dataset.getInode()).isEmpty()) {
        throw new DelaException(RESTCodes.DelaErrorCode.DATASET_EMPTY, Level.WARNING, DelaException.Source.LOCAL);
    }
    LOGGER.log(Settings.DELA_DEBUG, "{0} - writing manifest", dataset.getPublicDsId());
    ManifestJSON manifest = createManifest(project, dataset, user);
    Path manifestPath = manifestPath(dataset);
    delete(project, user, manifestPath);
    write(project, user, manifestPath, ManifestHelper.marshall(manifest));
    return manifest;
}

Example 4 with ManifestJSON

use of io.hops.hopsworks.dela.old_dto.ManifestJSON in project hopsworks by logicalclocks.

the class DelaWorkerController method startDownload.

public ManifestJSON startDownload(Project project, Users user, HopsworksTransferDTO.Download downloadDTO) throws DelaException, DatasetException, ProvenanceException, IOException {
    delaStateCtrl.checkDelaAvailable();
    Dataset dataset = delaDatasetCtrl.download(project, user, downloadDTO.getPublicDSId(), downloadDTO.getName());
    try {
        delaCtrlStartDownload(project, dataset, user, downloadDTO);
    } catch (DelaException tpe) {
        delaDatasetCtrl.delete(project, dataset, user);
        throw tpe;
    }
    ManifestJSON manifest = delaHdfsCtrl.readManifest(project, dataset, user);
    delaDatasetCtrl.updateDescription(project, dataset, manifest.getDatasetDescription());
    return manifest;
}

Example 5 with ManifestJSON

use of io.hops.hopsworks.dela.old_dto.ManifestJSON in project hopsworks by logicalclocks.

the class ManifestHelper method unmarshall.

public static ManifestJSON unmarshall(byte[] jsonByte) throws DelaException {
    String jsonString;
    try {
        jsonString = new String(jsonByte, "UTF-8");
    } catch (UnsupportedEncodingException ex) {
        throw new DelaException(RESTCodes.DelaErrorCode.MANIFEST_ENCODING_ERROR, Level.SEVERE, DelaException.Source.LOCAL, null, ex.getMessage(), ex);
    }
    ManifestJSON manifest = new Gson().fromJson(jsonString, ManifestJSON.class);
    return manifest;
}