Examples with Artifact ai.djl.repository.Artifact used on opensource projects

Example 11 with Artifact

use of ai.djl.repository.Artifact in project djl by deepjavalibrary.

the class ModelZoo method listModels.

/**
 * Returns the available {@link Application} and their model artifact metadata.
 *
 * @param criteria the requirements for the model
 * @return the available {@link Application} and their model artifact metadata
 * @throws IOException if failed to download to repository metadata
 * @throws ModelNotFoundException if failed to parse repository metadata
 */
public static Map<Application, List<Artifact>> listModels(Criteria<?, ?> criteria) throws IOException, ModelNotFoundException {
    String artifactId = criteria.getArtifactId();
    ModelZoo modelZoo = criteria.getModelZoo();
    String groupId = criteria.getGroupId();
    String engine = criteria.getEngine();
    Application application = criteria.getApplication();
    @SuppressWarnings("PMD.UseConcurrentHashMap") Map<Application, List<Artifact>> models = new TreeMap<>(Comparator.comparing(Application::getPath));
    for (ModelZoo zoo : listModelZoo()) {
        if (modelZoo != null) {
            if (groupId != null && !modelZoo.getGroupId().equals(groupId)) {
                continue;
            }
            Set<String> supportedEngine = modelZoo.getSupportedEngines();
            if (engine != null && !supportedEngine.contains(engine)) {
                continue;
            }
        }
        List<ModelLoader> list = zoo.getModelLoaders();
        for (ModelLoader loader : list) {
            Application app = loader.getApplication();
            String loaderArtifactId = loader.getArtifactId();
            if (artifactId != null && !artifactId.equals(loaderArtifactId)) {
                // filter out by model loader artifactId
                continue;
            }
            if (application != Application.UNDEFINED && app != Application.UNDEFINED && !app.matches(application)) {
                // filter out ModelLoader by application
                continue;
            }
            final List<Artifact> artifacts = loader.listModels();
            models.compute(app, (key, val) -> {
                if (val == null) {
                    val = new ArrayList<>();
                }
                val.addAll(artifacts);
                return val;
            });
        }
    }
    return models;
}

Example 12 with Artifact

use of ai.djl.repository.Artifact in project djl by deepjavalibrary.

the class HdfsRepository method resolve.

/**
 * {@inheritDoc}
 */
@Override
public Artifact resolve(MRL mrl, Map<String, String> filter) throws IOException {
    Metadata m = locate(mrl);
    if (m == null) {
        return null;
    }
    List<Artifact> artifacts = m.getArtifacts();
    if (artifacts.isEmpty()) {
        return null;
    }
    return artifacts.get(0);
}

Example 13 with Artifact

use of ai.djl.repository.Artifact in project djl by deepjavalibrary.

the class HdfsRepository method listFiles.

private Artifact listFiles() throws IOException {
    FileSystem fs = FileSystem.get(uri, config);
    FileStatus[] status = fs.listStatus(new org.apache.hadoop.fs.Path(prefix));
    if (status == null || status.length == 0) {
        return null;
    }
    Artifact artifact = new Artifact();
    artifact.setName(modelName);
    artifact.getArguments().putAll(arguments);
    Map<String, Artifact.Item> files = new ConcurrentHashMap<>();
    artifact.setFiles(files);
    if (isDirectory) {
        Path fullPath = Paths.get(prefix);
        for (FileStatus st : status) {
            Artifact.Item item = new Artifact.Item();
            String key = st.getPath().getName();
            if (!key.endsWith("/")) {
                item.setUri(fullPath.resolve(key).toString());
                item.setSize(st.getLen());
                item.setArtifact(artifact);
                if ("dir".equals(item.getType())) {
                    // avoid creating extra folder
                    item.setName("");
                }
                files.put(key, item);
            }
        }
    } else {
        Artifact.Item item = new Artifact.Item();
        item.setUri(prefix);
        // avoid creating extra folder
        item.setName("");
        item.setArtifact(artifact);
        item.setSize(status[0].getLen());
        files.put(artifactId, item);
    }
    return artifact;
}

Example 14 with Artifact

use of ai.djl.repository.Artifact in project djl by deepjavalibrary.

the class HdfsRepository method getMetadata.

private synchronized Metadata getMetadata() throws IOException {
    if (resolved) {
        return metadata;
    }
    resolved = true;
    Artifact artifact = listFiles();
    if (artifact == null) {
        logger.debug("No object found in hdfs: " + uri);
        return null;
    }
    metadata = new Metadata.MatchAllMetadata();
    String hash = md5hash(uri.resolve(prefix).toString());
    MRL mrl = model(Application.UNDEFINED, DefaultModelZoo.GROUP_ID, hash);
    metadata.setRepositoryUri(mrl.toURI());
    metadata.setArtifactId(artifactId);
    metadata.setArtifacts(Collections.singletonList(artifact));
    return metadata;
}

Example 15 with Artifact

use of ai.djl.repository.Artifact in project djl by deepjavalibrary.

the class HdfsRepositoryTest method testAccessDeny.

@Test
public void testAccessDeny() throws IOException {
    int port = miniDfs.getNameNodePort();
    Repository repo = Repository.newInstance("hdfs", "hdfs://localhost:" + port + "/non-exists");
    List<MRL> list = repo.getResources();
    Assert.assertTrue(list.isEmpty());
    MRL mrl = repo.model(Application.UNDEFINED, "ai.djl.localmodelzoo", "mlp");
    Artifact artifact = repo.resolve(mrl, null);
    Assert.assertNull(artifact);
}