Search in sources :

Example 16 with ArtifactNotFoundException

use of io.cdap.cdap.common.ArtifactNotFoundException in project cdap by caskdata.

the class ArtifactStoreTest method testGetPluginsByParentArtifactRanges.

@Test
public void testGetPluginsByParentArtifactRanges() throws Exception {
    ArtifactRange parentArtifacts1 = new ArtifactRange(NamespaceId.DEFAULT.getNamespace(), "parent1", new ArtifactVersion("1.0.0"), new ArtifactVersion("5.0.0"));
    // we have 2 plugins of type A and 2 plugins of type B
    PluginClass pluginA1 = PluginClass.builder().setName("p1").setType("A").setDescription("desc").setClassName("c.p1").setConfigFieldName("cfg").setProperties(ImmutableMap.of("threshold", new PluginPropertyField("thresh", "description", "double", true, false), "retry", new PluginPropertyField("retries", "description", "int", false, false))).build();
    PluginClass pluginA2 = PluginClass.builder().setName("p2").setType("A").setDescription("desc").setClassName("c.p2").setConfigFieldName("conf").setProperties(ImmutableMap.of("stream", new PluginPropertyField("stream", "description", "string", true, false))).build();
    // add artifacts
    // not interested in artifact contents for this test, using some dummy value
    String contents = "0";
    // write parent artifacts
    List<String> parentArtifactsVersions = ImmutableList.of("1.0.0", "1.2.1", "2.0.0", "3.0.0", "4.0.0");
    for (String artifactVersion : parentArtifactsVersions) {
        Id.Artifact parentArtifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "parent1", artifactVersion);
        ArtifactMeta parentMeta = new ArtifactMeta(ArtifactClasses.builder().build());
        writeArtifact(parentArtifactId, parentMeta, contents);
    }
    // artifact artifactX-1.0.0 contains plugin A1
    Id.Artifact artifactXv100 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactX", "1.0.0");
    ArtifactMeta metaXv100 = new ArtifactMeta(ArtifactClasses.builder().addPlugin(pluginA1).build(), ImmutableSet.of(parentArtifacts1));
    writeArtifact(artifactXv100, metaXv100, contents);
    ArtifactDescriptor artifactXv100Info = artifactStore.getArtifact(artifactXv100).getDescriptor();
    // artifact artifactX-1.1.0 contains plugin A1
    Id.Artifact artifactXv110 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactX", "1.1.0");
    ArtifactMeta metaXv110 = new ArtifactMeta(ArtifactClasses.builder().addPlugin(pluginA1).build(), ImmutableSet.of(parentArtifacts1));
    writeArtifact(artifactXv110, metaXv110, contents);
    ArtifactDescriptor artifactXv110Info = artifactStore.getArtifact(artifactXv110).getDescriptor();
    // artifact artifactX-2.0.0 contains plugins A1 and A2
    Id.Artifact artifactXv200 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactX", "2.0.0");
    ArtifactMeta metaXv200 = new ArtifactMeta(ArtifactClasses.builder().addPlugins(pluginA1, pluginA2).build(), ImmutableSet.of(parentArtifacts1));
    writeArtifact(artifactXv200, metaXv200, contents);
    ArtifactDescriptor artifactXv200Info = artifactStore.getArtifact(artifactXv200).getDescriptor();
    ArtifactRange parentArtifactsrange1 = new ArtifactRange(NamespaceId.DEFAULT.getNamespace(), "parent1", new ArtifactVersion("3.0.0"), new ArtifactVersion("5.0.0"));
    // artifact artifactZ-2.0.0 contains plugins A1, A2
    Id.Artifact artifactZv200 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactZ", "2.0.0");
    ArtifactMeta metaZv200 = new ArtifactMeta(ArtifactClasses.builder().addPlugins(pluginA1, pluginA2).build(), ImmutableSet.of(parentArtifactsrange1));
    writeArtifact(artifactZv200, metaZv200, contents);
    ArtifactDescriptor artifactZv200Info = artifactStore.getArtifact(artifactZv200).getDescriptor();
    // artifact written with this range should not come up as their parent range is out of the parent artifact range.
    ArtifactRange parentArtifactsOutOfRange1 = new ArtifactRange(NamespaceId.DEFAULT.getNamespace(), "parent1", new ArtifactVersion("5.0.0"), new ArtifactVersion("8.0.0"));
    // artifact artifactZ-2.0.0 contains plugins A1, A2, B1, and B2
    Id.Artifact artifactZv300 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactZ", "3.0.0");
    ArtifactMeta metaZv300 = new ArtifactMeta(ArtifactClasses.builder().addPlugins(pluginA1, pluginA2).build(), ImmutableSet.of(parentArtifactsOutOfRange1));
    writeArtifact(artifactZv300, metaZv300, contents);
    Map<ArtifactDescriptor, PluginClass> expectedMap = Maps.newHashMap();
    expectedMap.put(artifactXv100Info, pluginA1);
    expectedMap.put(artifactXv110Info, pluginA1);
    expectedMap.put(artifactXv200Info, pluginA1);
    expectedMap.put(artifactZv200Info, pluginA1);
    Assert.assertEquals(expectedMap, artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifacts1, "A", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED));
    // test limited number
    Assert.assertEquals(ImmutableMap.of(artifactXv100Info, pluginA1), artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifacts1, "A", "p1", null, 1, ArtifactSortOrder.UNORDERED));
    // test DESC order
    Assert.assertEquals(expectedMap, new TreeMap<>(artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifacts1, "A", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.DESC)).descendingMap());
    ArtifactRange parentArtifactsSub1 = new ArtifactRange(NamespaceId.DEFAULT.getNamespace(), "parent1", new ArtifactVersion("1.1.0"), new ArtifactVersion("2.0.0"));
    expectedMap = Maps.newHashMap();
    expectedMap.put(artifactXv100Info, pluginA1);
    expectedMap.put(artifactXv110Info, pluginA1);
    expectedMap.put(artifactXv200Info, pluginA1);
    // artifactZv200Info wont be here, as the parent range 3.0.0-5.0.0 for artifactZv200 plugin
    // wont match the 1.2.1 parent artifact version
    Assert.assertEquals(expectedMap, artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactsSub1, "A", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED));
    expectedMap = Maps.newHashMap();
    expectedMap.put(artifactXv200Info, pluginA2);
    expectedMap.put(artifactZv200Info, pluginA2);
    Assert.assertEquals(expectedMap, artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifacts1, "A", "p2", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED));
    ArtifactRange parentArtifactsSub2 = new ArtifactRange(NamespaceId.DEFAULT.getNamespace(), "parent1", new ArtifactVersion("5.0.0"), new ArtifactVersion("10.0.0"));
    try {
        artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactsSub2, "A", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED);
        Assert.fail("Get plugin class for invalid range should not retrun result");
    } catch (ArtifactNotFoundException e) {
    // no-op
    }
}
Also used : ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) TreeMap(java.util.TreeMap) PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) PluginClass(io.cdap.cdap.api.plugin.PluginClass) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Test(org.junit.Test)

Example 17 with ArtifactNotFoundException

use of io.cdap.cdap.common.ArtifactNotFoundException in project cdap by cdapio.

the class RemotePluginFinder method findPlugin.

@Override
public Map.Entry<ArtifactDescriptor, PluginClass> findPlugin(NamespaceId pluginNamespaceId, ArtifactId parentArtifactId, String pluginType, String pluginName, PluginSelector selector) throws PluginNotExistsException {
    try {
        return Retries.callWithRetries(() -> {
            List<PluginInfo> infos = getPlugins(pluginNamespaceId, parentArtifactId, pluginType, pluginName);
            if (infos.isEmpty()) {
                throw new PluginNotExistsException(pluginNamespaceId, pluginType, pluginName);
            }
            SortedMap<io.cdap.cdap.api.artifact.ArtifactId, PluginClass> plugins = new TreeMap<>();
            for (PluginInfo info : infos) {
                ArtifactSummary artifactSummary = info.getArtifact();
                io.cdap.cdap.api.artifact.ArtifactId pluginArtifactId = new io.cdap.cdap.api.artifact.ArtifactId(artifactSummary.getName(), new ArtifactVersion(artifactSummary.getVersion()), artifactSummary.getScope());
                PluginClass pluginClass = PluginClass.builder().setName(info.getName()).setType(info.getType()).setDescription(info.getDescription()).setClassName(info.getClassName()).setProperties(info.getProperties()).setConfigFieldName(info.getConfigFieldName()).build();
                plugins.put(pluginArtifactId, pluginClass);
            }
            Map.Entry<io.cdap.cdap.api.artifact.ArtifactId, PluginClass> selected = selector.select(plugins);
            if (selected == null) {
                throw new PluginNotExistsException(pluginNamespaceId, pluginType, pluginName);
            }
            Location artifactLocation = getArtifactLocation(Artifacts.toProtoArtifactId(pluginNamespaceId, selected.getKey()));
            return Maps.immutableEntry(new ArtifactDescriptor(pluginNamespaceId.getEntityName(), selected.getKey(), artifactLocation), selected.getValue());
        }, retryStrategy);
    } catch (PluginNotExistsException e) {
        throw e;
    } catch (ArtifactNotFoundException e) {
        throw new PluginNotExistsException(pluginNamespaceId, pluginType, pluginName);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}
Also used : ArtifactId(io.cdap.cdap.proto.id.ArtifactId) TreeMap(java.util.TreeMap) ServiceUnavailableException(io.cdap.cdap.common.ServiceUnavailableException) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) IOException(java.io.IOException) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) ArtifactSummary(io.cdap.cdap.api.artifact.ArtifactSummary) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) PluginInfo(io.cdap.cdap.proto.artifact.PluginInfo) PluginClass(io.cdap.cdap.api.plugin.PluginClass) Map(java.util.Map) TreeMap(java.util.TreeMap) SortedMap(java.util.SortedMap) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Location(org.apache.twill.filesystem.Location)

Example 18 with ArtifactNotFoundException

use of io.cdap.cdap.common.ArtifactNotFoundException in project cdap by cdapio.

the class ArtifactHttpHandlerTestBase method getArtifactLocationPath.

/**
 * Get the location path of the given artifact.
 */
String getArtifactLocationPath(ArtifactId artifactId) throws ArtifactNotFoundException, IOException {
    URL endpoint = getEndPoint(String.format("%s/namespaces/%s/artifacts/%s/versions/%s/location", Constants.Gateway.INTERNAL_API_VERSION_3, artifactId.getNamespace(), artifactId.getArtifact(), artifactId.getVersion())).toURL();
    HttpResponse httpResponse = HttpRequests.execute(HttpRequest.get(endpoint).build(), new DefaultHttpRequestConfig(false));
    int responseCode = httpResponse.getResponseCode();
    if (responseCode == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new ArtifactNotFoundException(artifactId);
    }
    Assert.assertEquals(HttpURLConnection.HTTP_OK, responseCode);
    return httpResponse.getResponseBodyAsString();
}
Also used : DefaultHttpRequestConfig(io.cdap.cdap.common.http.DefaultHttpRequestConfig) HttpResponse(io.cdap.common.http.HttpResponse) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) URL(java.net.URL)

Example 19 with ArtifactNotFoundException

use of io.cdap.cdap.common.ArtifactNotFoundException in project cdap by cdapio.

the class AbstractProgramRuntimeService method createPluginSnapshot.

/**
 * Return the copy of the {@link ProgramOptions} including locations of plugin artifacts in it.
 * @param options the {@link ProgramOptions} in which the locations of plugin artifacts needs to be included
 * @param programId Id of the Program
 * @param tempDir Temporary Directory to create the plugin artifact snapshot
 * @param appSpec program's Application Specification
 * @return the copy of the program options with locations of plugin artifacts included in them
 */
private ProgramOptions createPluginSnapshot(ProgramOptions options, ProgramId programId, File tempDir, @Nullable ApplicationSpecification appSpec) throws Exception {
    // appSpec is null in an unit test
    if (appSpec == null) {
        return options;
    }
    Set<String> files = Sets.newHashSet();
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    builder.putAll(options.getArguments().asMap());
    for (Map.Entry<String, Plugin> pluginEntry : appSpec.getPlugins().entrySet()) {
        Plugin plugin = pluginEntry.getValue();
        File destFile = new File(tempDir, Artifacts.getFileName(plugin.getArtifactId()));
        // Skip if the file has already been copied.
        if (!files.add(destFile.getName())) {
            continue;
        }
        try {
            ArtifactId artifactId = Artifacts.toProtoArtifactId(programId.getNamespaceId(), plugin.getArtifactId());
            copyArtifact(artifactId, noAuthArtifactRepository.getArtifact(Id.Artifact.fromEntityId(artifactId)), destFile);
        } catch (ArtifactNotFoundException e) {
            throw new IllegalArgumentException(String.format("Artifact %s could not be found", plugin.getArtifactId()), e);
        }
    }
    LOG.debug("Plugin artifacts of {} copied to {}", programId, tempDir.getAbsolutePath());
    builder.put(ProgramOptionConstants.PLUGIN_DIR, tempDir.getAbsolutePath());
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(builder.build()), options.getUserArguments(), options.isDebug());
}
Also used : ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ImmutableMap(com.google.common.collect.ImmutableMap) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) File(java.io.File) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Plugin(io.cdap.cdap.api.plugin.Plugin)

Example 20 with ArtifactNotFoundException

use of io.cdap.cdap.common.ArtifactNotFoundException in project cdap by cdapio.

the class ArtifactStore method getPluginsInArtifact.

private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(StructuredTable artifactDataTable, Id.Artifact artifactId, Predicate<PluginClass> filter) throws ArtifactNotFoundException, IOException {
    SortedMap<ArtifactDescriptor, Set<PluginClass>> result = new TreeMap<>();
    // Make sure the artifact exists
    ArtifactCell artifactCell = new ArtifactCell(artifactId);
    Optional<StructuredRow> row = artifactDataTable.read(artifactCell.keys);
    if (!row.isPresent()) {
        throw new ArtifactNotFoundException(artifactId.toEntityId());
    }
    // include any plugin classes that are inside the artifact itself and is accepted by the filter
    ArtifactData artifactData = GSON.fromJson(row.get().getString(StoreDefinition.ArtifactStore.ARTIFACT_DATA_FIELD), ArtifactData.class);
    Set<PluginClass> plugins = artifactData.meta.getClasses().getPlugins().stream().filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
    if (!plugins.isEmpty()) {
        Location location = Locations.getLocationFromAbsolutePath(locationFactory, artifactData.getLocationPath());
        ArtifactDescriptor descriptor = new ArtifactDescriptor(artifactId.getNamespace().getId(), artifactId.toArtifactId(), location);
        result.put(descriptor, plugins);
    }
    return result;
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) TreeMap(java.util.TreeMap) PluginClass(io.cdap.cdap.api.plugin.PluginClass) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Location(org.apache.twill.filesystem.Location)

Aggregations

ArtifactNotFoundException (io.cdap.cdap.common.ArtifactNotFoundException)62 HttpResponse (io.cdap.common.http.HttpResponse)24 URL (java.net.URL)20 IOException (java.io.IOException)18 ArtifactRange (io.cdap.cdap.api.artifact.ArtifactRange)16 ArtifactId (io.cdap.cdap.proto.id.ArtifactId)16 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)16 ArtifactVersion (io.cdap.cdap.api.artifact.ArtifactVersion)14 HttpRequest (io.cdap.common.http.HttpRequest)14 ArtifactSummary (io.cdap.cdap.api.artifact.ArtifactSummary)12 PluginClass (io.cdap.cdap.api.plugin.PluginClass)12 StructuredRow (io.cdap.cdap.spi.data.StructuredRow)12 Location (org.apache.twill.filesystem.Location)12 BadRequestException (io.cdap.cdap.common.BadRequestException)10 File (java.io.File)10 Test (org.junit.Test)10 StructuredTable (io.cdap.cdap.spi.data.StructuredTable)9 ApplicationClass (io.cdap.cdap.api.artifact.ApplicationClass)6 ApplicationNotFoundException (io.cdap.cdap.common.ApplicationNotFoundException)6 Id (io.cdap.cdap.common.id.Id)6