Examples with PackageMetadata org.commonjava.indy.pkg.npm.model.PackageMetadata used on opensource projects

Example 1 with PackageMetadata

use of org.commonjava.indy.pkg.npm.model.PackageMetadata in project indy by Commonjava.

the class PackageMetadataGenerator method writePackageMetadata.

private boolean writePackageMetadata(List<StoreResource> firstLevelFiles, ArtifactStore store, String path, EventMetadata eventMetadata) throws IndyWorkflowException {
    logger.debug("writePackageMetadata, firstLevelFiles:{}, store:{}", firstLevelFiles, store.getKey());
    // Parse the path of the tar (e.g.: jquery/-/jquery-7.6.1.tgz or @types/jquery/-/jquery-2.2.3.tgz)
    // to get the version, then try to get the version metadata by the path (@scoped/)package/version
    List<PackagePath> packagePaths = firstLevelFiles.stream().map((res) -> new PackagePath(res.getPath())).sorted(Comparator.comparing(PackagePath::getVersion)).collect(Collectors.toList());
    if (packagePaths.size() == 0) {
        return false;
    }
    final Transfer metadataFile = fileManager.getTransfer(store, path);
    final PackageMetadata packageMetadata = new PackageMetadata();
    final IndyObjectMapper mapper = new IndyObjectMapper(true);
    List<String> keywords = new ArrayList<>();
    DistTag distTags = new DistTag();
    Map<String, VersionMetadata> versions = new LinkedHashMap<>();
    PackagePath latest = packagePaths.get(packagePaths.size() - 1);
    for (PackagePath packagePath : packagePaths) {
        String versionPath = packagePath.getVersionPath();
        logger.debug("Retrieving the version file {} from store {}", versionPath, store);
        Transfer metaFile = fileManager.retrieveRaw(store, versionPath, eventMetadata);
        if (metaFile == null) {
            // The metadata file (@scoped/)package/version for the specific version is missing, still need to extract it from tarball
            String tarPath = packagePath.getTarPath();
            Transfer tar = fileManager.retrieveRaw(store, tarPath, eventMetadata);
            if (tar == null) {
                logger.warn("Tarball file {} is missing in the store {}.", tarPath, store.getKey());
                continue;
            }
            logger.info("Extracting package metadata package.json from tarball {} and store it in {}/{}", tarPath, store.getKey(), versionPath);
            metaFile = extractMetaFileFromTarballAndStore(store, versionPath, tar);
            if (metaFile == null) {
                logger.warn("Package metadata is missing in tarball {}/{}.", store.getKey(), tarPath);
                continue;
            }
        }
        try (InputStream input = metaFile.openInputStream()) {
            VersionMetadata versionMetadata = mapper.readValue(input, VersionMetadata.class);
            if (versionMetadata == null) {
                continue;
            }
            versions.put(versionMetadata.getVersion(), versionMetadata);
            if (versionMetadata.getKeywords() != null) {
                for (String keyword : versionMetadata.getKeywords()) {
                    if (!keywords.contains(keyword)) {
                        keywords.add(keyword);
                    }
                }
            }
            // Set couple of attributes based on the latest version metadata
            if (packagePath.getVersion().equals(latest.getVersion())) {
                packageMetadata.setName(versionMetadata.getName());
                packageMetadata.setDescription(versionMetadata.getDescription());
                packageMetadata.setAuthor(versionMetadata.getAuthor());
                if (versionMetadata.getLicense() != null) {
                    packageMetadata.setLicense(versionMetadata.getLicense());
                }
                packageMetadata.setRepository(versionMetadata.getRepository());
                packageMetadata.setBugs(versionMetadata.getBugs());
                distTags.setLatest(versionMetadata.getVersion());
            }
            // Generate tarball url if missing
            if (versionMetadata.getDist() == null) {
                // here we use mock host. indy will amend it with the right hostname
                String tarball = "http://indy/" + packagePath.getTarPath();
                // logger.debug( "Generate dist tarball: {}", tarball );
                versionMetadata.setDist(new Dist(null, tarball));
            }
        } catch (IOException e) {
            logger.error("Get the version metadata error from path {}", versionPath, e);
            throw new IndyWorkflowException("Get the version metadata error from path {}", versionPath);
        }
    }
    if (!keywords.isEmpty()) {
        packageMetadata.setKeywords(keywords);
    }
    versions = sortVersions(versions);
    packageMetadata.setVersions(versions);
    packageMetadata.setDistTags(distTags);
    OutputStream stream = null;
    try {
        String output = mapper.writeValueAsString(packageMetadata);
        stream = metadataFile.openOutputStream(TransferOperation.GENERATE, true, eventMetadata);
        stream.write(output.getBytes(StandardCharsets.UTF_8));
    } catch (IOException e) {
        throw new IndyWorkflowException("Generating package metadata failure in store {}", store.getKey());
    } finally {
        closeQuietly(stream);
    }
    logger.debug("writePackageMetadata, DONE, store: {}", store.getKey());
    return true;
}

Example 2 with PackageMetadata

use of org.commonjava.indy.pkg.npm.model.PackageMetadata in project indy by Commonjava.

the class PackageMetadataMerger method merge.

public byte[] merge(final Collection<Transfer> sources, final Group group, final String path) {
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.debug("Generating merged metadata in: {}:{}", group.getKey(), path);
    InputStream stream = null;
    boolean merged = false;
    final PackageMetadata packageMetadata = new PackageMetadata();
    for (final Transfer src : sources) {
        if (!src.exists()) {
            continue;
        }
        try {
            stream = src.openInputStream();
            String content = IOUtils.toString(stream);
            logger.debug("Adding in metadata content from: {}\n\n{}\n\n", src, content);
            PackageMetadata md = mapper.readValue(content, PackageMetadata.class);
            merged = packageMetadata.merge(md, true) || merged;
        } catch (final IOException e) {
            final StoreKey key = getKey(src);
            logger.error(String.format("Cannot read metadata: %s from artifact-store: %s. Reason: %s", src.getPath(), key, e.getMessage()), e);
        } finally {
            closeQuietly(stream);
        }
    }
    if (metadataProviders != null) {
        for (PackageMetadataProvider provider : metadataProviders) {
            try {
                PackageMetadata toMerge = provider.getMetadata(group.getKey(), path);
                if (toMerge != null) {
                    merged = packageMetadata.merge(toMerge, true) || merged;
                }
            } catch (IndyWorkflowException e) {
                logger.error(String.format("Cannot read metadata: %s from metadata provider: %s. Reason: %s", path, provider.getClass().getSimpleName(), e.getMessage()), e);
            }
        }
    }
    if (merged) {
        String output = null;
        try {
            output = mapper.writeValueAsString(packageMetadata);
        } catch (JsonProcessingException e) {
            logger.error(String.format("Cannot convert from metadata: %s to String. Reason: %s", packageMetadata, e.getMessage()), e);
        }
        return output.getBytes();
    }
    return null;
}

Example 3 with PackageMetadata

use of org.commonjava.indy.pkg.npm.model.PackageMetadata in project indy by Commonjava.

the class PackageMetadataMerger method merge.

public InputStream merge(final Transfer... sources) {
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.debug("[NPM] Generating merged metadata when publish");
    InputStream stream = null;
    final PackageMetadata packageMetadata = new PackageMetadata();
    for (final Transfer src : sources) {
        if (!src.exists()) {
            continue;
        }
        try {
            stream = src.openInputStream();
            String content = IOUtils.toString(stream);
            logger.debug("[NPM] Adding in metadata content from: {}\n\n{}\n\n", src, content);
            PackageMetadata md = mapper.readValue(content, PackageMetadata.class);
            packageMetadata.merge(md, false);
        } catch (final IOException e) {
            final StoreKey key = getKey(src);
            logger.error(String.format("[NPM] Cannot read metadata: %s from artifact-store: %s. Reason: %s", src.getPath(), key, e.getMessage()), e);
        } finally {
            closeQuietly(stream);
        }
    }
    String output = null;
    try {
        output = mapper.writeValueAsString(packageMetadata);
    } catch (JsonProcessingException e) {
        logger.error(String.format("[NPM] Cannot convert from metadata: %s to String. Reason: %s", packageMetadata, e.getMessage()), e);
    }
    if (output != null) {
        return IOUtils.toInputStream(output);
    }
    return null;
}

Example 4 with PackageMetadata

use of org.commonjava.indy.pkg.npm.model.PackageMetadata in project indy by Commonjava.

the class PackageMetadataGeneratorTest method verifyScopedMetadata.

private void verifyScopedMetadata(Transfer metadataFile) throws Exception {
    final IndyObjectMapper mapper = new IndyObjectMapper(true);
    try (InputStream input = metadataFile.openInputStream()) {
        PackageMetadata packageMetadata = mapper.readValue(input, PackageMetadata.class);
        assertNotNull(packageMetadata);
        assertEquals(2, packageMetadata.getVersions().size());
        assertEquals("Unexpected package name.", "@babel/core", packageMetadata.getName());
        assertEquals("Unexpected latest version.", "7.7.7", packageMetadata.getDistTags().getLatest());
        // verify tarball not null
        for (VersionMetadata versionMetadata : packageMetadata.getVersions().values()) {
            assertNotNull(versionMetadata.getDist().getTarball());
        }
    }
}

Example 5 with PackageMetadata

use of org.commonjava.indy.pkg.npm.model.PackageMetadata in project indy by Commonjava.

the class PackageMetadataMergerTest method mergeTwoSimplePackageMetadataFiles.

@Test
public void mergeTwoSimplePackageMetadataFiles() throws Exception {
    String path = "jquery";
    HostedRepository h1 = new HostedRepository(NPM_PKG_KEY, "test-hosted-1");
    HostedRepository h2 = new HostedRepository(NPM_PKG_KEY, "test-hosted-2");
    Transfer t1 = cacheProvider.getTransfer(new ConcreteResource(LocationUtils.toLocation(h1), path));
    initTestData(t1, VERSION_META + "package-1.json");
    Transfer t2 = cacheProvider.getTransfer(new ConcreteResource(LocationUtils.toLocation(h2), path));
    initTestData(t2, VERSION_META + "package-2.json");
    Group g = new Group(NPM_PKG_KEY, "test-group", h1.getKey(), h2.getKey());
    List<Transfer> sources = Arrays.asList(t1, t2);
    byte[] output = new PackageMetadataMerger(Collections.emptyList(), mapper).merge(sources, g, path);
    PackageMetadata merged = mapper.readValue(IOUtils.toString(new ByteArrayInputStream(output)), PackageMetadata.class);
    // normal object fields merging verification
    assertThat(merged.getName(), equalTo("jquery"));
    assertThat(merged.getAuthor().getName(), equalTo("JS Foundation and other contributors"));
    assertThat(merged.getRepository().getType(), equalTo("git"));
    assertThat(merged.getRepository().getUrl(), equalTo("git+https://github.com/jquery/jquery.git"));
    assertThat(merged.getReadmeFilename(), equalTo("README.md"));
    assertThat(merged.getHomepage(), equalTo("https://jquery.com"));
    assertThat(merged.getBugs().getUrl(), equalTo("https://github.com/jquery/jquery/issues"));
    assertThat(merged.getLicense().getType(), equalTo("MIT"));
    // dist-tags object merging verification
    assertThat(merged.getDistTags().getBeta(), equalTo("3.2.1-beta.1"));
    assertThat(merged.getDistTags().getLatest(), equalTo("3.2.1"));
    // versions map merging verification
    Map<String, VersionMetadata> versions = merged.getVersions();
    assertThat(versions, notNullValue());
    assertThat(versions.size(), equalTo(2));
    assertThat(versions.get("1.5.1").getVersion(), equalTo("1.5.1"));
    assertThat(versions.get("1.6.2").getVersion(), equalTo("1.6.2"));
    // maintainers list merging verification
    List<UserInfo> maintainers = merged.getMaintainers();
    assertThat(maintainers, notNullValue());
    assertThat(maintainers.size(), equalTo(3));
    assertThat(maintainers.get(0).getName(), equalTo("dmethvin"));
    assertThat(maintainers.get(1).getName(), equalTo("mgol"));
    assertThat(maintainers.get(2).getName(), equalTo("scott.gonzalez"));
    // time map merging verification
    Map<String, String> times = merged.getTime();
    assertThat(times, notNullValue());
    assertThat(times.size(), equalTo(8));
    assertThat(times.get("modified"), equalTo("2017-05-23T10:57:14.309Z"));
    assertThat(times.get("created"), equalTo("2011-04-19T07:19:56.392Z"));
    // users map merging verification
    Map<String, Boolean> users = merged.getUsers();
    assertThat(users, notNullValue());
    assertThat(users.size(), equalTo(10));
    assertThat(users.get("fgribreau"), equalTo(true));
    // keywords list merging verification
    List<String> keywords = merged.getKeywords();
    assertThat(keywords, notNullValue());
    assertThat(keywords.size(), equalTo(4));
    assertThat(keywords.contains("javascript"), equalTo(true));
}