Examples with DistTag org.commonjava.indy.pkg.npm.model.DistTag used on opensource projects

Example 1 with DistTag

use of org.commonjava.indy.pkg.npm.model.DistTag in project indy by Commonjava.

the class PackageMetadataGenerator method writePackageMetadata.

private boolean writePackageMetadata(List<StoreResource> firstLevelFiles, ArtifactStore store, String path, EventMetadata eventMetadata) throws IndyWorkflowException {
    logger.debug("writePackageMetadata, firstLevelFiles:{}, store:{}", firstLevelFiles, store.getKey());
    // Parse the path of the tar (e.g.: jquery/-/jquery-7.6.1.tgz or @types/jquery/-/jquery-2.2.3.tgz)
    // to get the version, then try to get the version metadata by the path (@scoped/)package/version
    List<PackagePath> packagePaths = firstLevelFiles.stream().map((res) -> new PackagePath(res.getPath())).sorted(Comparator.comparing(PackagePath::getVersion)).collect(Collectors.toList());
    if (packagePaths.size() == 0) {
        return false;
    }
    final Transfer metadataFile = fileManager.getTransfer(store, path);
    final PackageMetadata packageMetadata = new PackageMetadata();
    final IndyObjectMapper mapper = new IndyObjectMapper(true);
    List<String> keywords = new ArrayList<>();
    DistTag distTags = new DistTag();
    Map<String, VersionMetadata> versions = new LinkedHashMap<>();
    PackagePath latest = packagePaths.get(packagePaths.size() - 1);
    for (PackagePath packagePath : packagePaths) {
        String versionPath = packagePath.getVersionPath();
        logger.debug("Retrieving the version file {} from store {}", versionPath, store);
        Transfer metaFile = fileManager.retrieveRaw(store, versionPath, eventMetadata);
        if (metaFile == null) {
            // The metadata file (@scoped/)package/version for the specific version is missing, still need to extract it from tarball
            String tarPath = packagePath.getTarPath();
            Transfer tar = fileManager.retrieveRaw(store, tarPath, eventMetadata);
            if (tar == null) {
                logger.warn("Tarball file {} is missing in the store {}.", tarPath, store.getKey());
                continue;
            }
            logger.info("Extracting package metadata package.json from tarball {} and store it in {}/{}", tarPath, store.getKey(), versionPath);
            metaFile = extractMetaFileFromTarballAndStore(store, versionPath, tar);
            if (metaFile == null) {
                logger.warn("Package metadata is missing in tarball {}/{}.", store.getKey(), tarPath);
                continue;
            }
        }
        try (InputStream input = metaFile.openInputStream()) {
            VersionMetadata versionMetadata = mapper.readValue(input, VersionMetadata.class);
            if (versionMetadata == null) {
                continue;
            }
            versions.put(versionMetadata.getVersion(), versionMetadata);
            if (versionMetadata.getKeywords() != null) {
                for (String keyword : versionMetadata.getKeywords()) {
                    if (!keywords.contains(keyword)) {
                        keywords.add(keyword);
                    }
                }
            }
            // Set couple of attributes based on the latest version metadata
            if (packagePath.getVersion().equals(latest.getVersion())) {
                packageMetadata.setName(versionMetadata.getName());
                packageMetadata.setDescription(versionMetadata.getDescription());
                packageMetadata.setAuthor(versionMetadata.getAuthor());
                if (versionMetadata.getLicense() != null) {
                    packageMetadata.setLicense(versionMetadata.getLicense());
                }
                packageMetadata.setRepository(versionMetadata.getRepository());
                packageMetadata.setBugs(versionMetadata.getBugs());
                distTags.setLatest(versionMetadata.getVersion());
            }
            // Generate tarball url if missing
            if (versionMetadata.getDist() == null) {
                // here we use mock host. indy will amend it with the right hostname
                String tarball = "http://indy/" + packagePath.getTarPath();
                // logger.debug( "Generate dist tarball: {}", tarball );
                versionMetadata.setDist(new Dist(null, tarball));
            }
        } catch (IOException e) {
            logger.error("Get the version metadata error from path {}", versionPath, e);
            throw new IndyWorkflowException("Get the version metadata error from path {}", versionPath);
        }
    }
    if (!keywords.isEmpty()) {
        packageMetadata.setKeywords(keywords);
    }
    versions = sortVersions(versions);
    packageMetadata.setVersions(versions);
    packageMetadata.setDistTags(distTags);
    OutputStream stream = null;
    try {
        String output = mapper.writeValueAsString(packageMetadata);
        stream = metadataFile.openOutputStream(TransferOperation.GENERATE, true, eventMetadata);
        stream.write(output.getBytes(StandardCharsets.UTF_8));
    } catch (IOException e) {
        throw new IndyWorkflowException("Generating package metadata failure in store {}", store.getKey());
    } finally {
        closeQuietly(stream);
    }
    logger.debug("writePackageMetadata, DONE, store: {}", store.getKey());
    return true;
}

Example 2 with DistTag

use of org.commonjava.indy.pkg.npm.model.DistTag in project indy by Commonjava.

the class NPMMetadataTimeoutTest method test.

@Test
@Category(TimingDependent.class)
public void test() throws Exception {
    IndyObjectMapper mapper = new IndyObjectMapper(true);
    final PackageMetadata src = new PackageMetadata();
    final DistTag dts = new DistTag();
    dts.setBeta("1");
    src.setDistTags(dts);
    server.expect("GET", server.formatUrl(REPO, PATH_JQUERY), (req, resp) -> {
        resp.setStatus(200);
        mapper.writeValue(resp.getWriter(), src);
        resp.getWriter().flush();
    });
    server.expect("GET", server.formatUrl(REPO, PATH_BABEL_PARSER), (req, resp) -> {
        resp.setStatus(200);
        mapper.writeValue(resp.getWriter(), src);
        resp.getWriter().flush();
    });
    final RemoteRepository repo = new RemoteRepository(NPM_PKG_KEY, REPO, server.formatUrl(REPO));
    repo.setMetadataTimeoutSeconds(1);
    client.stores().create(repo, "adding npm remote repo", RemoteRepository.class);
    // First retrieval
    verifyMetadataBetaTag("1", PATH_JQUERY, repo);
    assertThat("Metadata not retrieved!", client.content().exists(repo.getKey(), PATH_JQUERY, true), equalTo(true));
    assertThat("Metadata not retrieved!", client.content().exists(repo.getKey(), PATH_JQUERY + "/package.json", true), equalTo(true));
    verifyMetadataBetaTag("1", PATH_BABEL_PARSER, repo);
    assertThat("Metadata not retrieved!", client.content().exists(repo.getKey(), PATH_BABEL_PARSER, true), equalTo(true));
    assertThat("Metadata not retrieved!", client.content().exists(repo.getKey(), PATH_BABEL_PARSER + "/package.json", true), equalTo(true));
    // wait for repo metadata timeout
    Thread.sleep(3000);
    assertThat("Metadata not cleaned up!", client.content().exists(repo.getKey(), PATH_JQUERY, true), equalTo(false));
    assertThat("Metadata not cleaned up!", client.content().exists(repo.getKey(), PATH_JQUERY + "/package.json", true), equalTo(false));
    assertThat("Metadata not cleaned up!", client.content().exists(repo.getKey(), PATH_BABEL_PARSER, true), equalTo(false));
    assertThat("Metadata not cleaned up!", client.content().exists(repo.getKey(), PATH_BABEL_PARSER + "/package.json", true), equalTo(false));
    logger.info("\n\n\n\nRE-REQUEST STARTS HERE\n\n\n\n");
    // Second retrieval
    dts.setBeta("2");
    verifyMetadataBetaTag("2", PATH_JQUERY, repo);
    verifyMetadataBetaTag("2", PATH_BABEL_PARSER, repo);
}

Example 3 with DistTag

use of org.commonjava.indy.pkg.npm.model.DistTag in project indy by Commonjava.

the class PackageMetadataMergerTest method mergeProviderWithDistTagVersionSorted.

@Test
public void mergeProviderWithDistTagVersionSorted() throws Exception {
    String path = "jquery";
    HostedRepository h1 = new HostedRepository(NPM_PKG_KEY, "test-hosted-1");
    Transfer t1 = cacheProvider.getTransfer(new ConcreteResource(LocationUtils.toLocation(h1), path));
    initTestData(t1, VERSION_META + "package-dist-tag-version.json");
    Group g = new Group(NPM_PKG_KEY, "test-group", h1.getKey());
    List<Transfer> sources = Arrays.asList(t1);
    PackageMetadata provided = new PackageMetadata("jquery");
    DistTag distTag = new DistTag();
    distTag.putTag(DistTag.LATEST, "3.2.3-rc.3");
    distTag.putTag(DistTag.STABLE, "2.0");
    distTag.putTag(DistTag.BETA, "3.2.1-beta.2");
    distTag.putTag(DistTag.DEV, "3.2.3-alpha");
    provided.setDistTags(distTag);
    TestPackageMetadataProvider testProvider = new TestPackageMetadataProvider(provided);
    byte[] output = new PackageMetadataMerger(Collections.singletonList(testProvider), mapper).merge(sources, g, path);
    PackageMetadata merged = mapper.readValue(IOUtils.toString(new ByteArrayInputStream(output)), PackageMetadata.class);
    assertThat(merged.getDistTags().getLatest(), equalTo("3.2.3-rc.3"));
    assertThat(merged.getDistTags().getStable(), equalTo("3.2.1"));
    assertThat(merged.getDistTags().getBeta(), equalTo("3.2.1-beta.2"));
    assertThat(merged.getDistTags().getDev(), equalTo("3.2.3-alpha"));
}

Example 4 with DistTag

use of org.commonjava.indy.pkg.npm.model.DistTag in project indy by Commonjava.

the class NPMMetadataRevisitTimeoutTest method test.

@Test
@Category(TimingDependent.class)
public void test() throws Exception {
    IndyObjectMapper mapper = new IndyObjectMapper(true);
    final PackageMetadata src = new PackageMetadata();
    final DistTag dts = new DistTag();
    dts.setBeta("1");
    src.setDistTags(dts);
    server.expect("GET", server.formatUrl(REPO, PATH), (req, resp) -> {
        resp.setStatus(200);
        mapper.writeValue(resp.getWriter(), src);
        resp.getWriter().flush();
    });
    final int METADATA_TIMEOUT_SECOND = 3;
    final int TIMEOUT_WAIT_MILLI_S = 1000;
    final RemoteRepository repo = new RemoteRepository(NPM_PKG_KEY, REPO, server.formatUrl(REPO));
    logger.info("\n\nCreate repo: {}\n\n", repo);
    repo.setMetadataTimeoutSeconds(METADATA_TIMEOUT_SECOND);
    client.stores().create(repo, "adding npm remote repo", RemoteRepository.class);
    logger.info("\n\nFirst retrieval\n\n");
    // First retrieval
    verifyMetadataBetaTag("1", repo);
    dts.setBeta("2");
    // wait for repo metadata timeout
    Thread.sleep(TIMEOUT_WAIT_MILLI_S * 2);
    logger.info("\n\nSecond retrieval: try to reset timeout\n\n");
    // Second retrieval when not timeout for first
    verifyMetadataBetaTag("1", repo);
    // wait for repo metadata timeout
    Thread.sleep(TIMEOUT_WAIT_MILLI_S * 2);
    assertThat("Metadata not cleaned up!", client.content().exists(repo.getKey(), PATH + "/package.json", true), equalTo(false));
    logger.info("\n\nThird retrieval: check if timeout happened\n\n");
    // Third retrieval
    verifyMetadataBetaTag("2", repo);
}