Search in sources :

Example 1 with RemoteClientFactory

use of io.cdap.cdap.common.internal.remote.RemoteClientFactory in project cdap by caskdata.

the class ArtifactLocalizerServiceTest method setupArtifactLocalizerService.

private ArtifactLocalizerService setupArtifactLocalizerService(CConfiguration cConf) {
    DiscoveryServiceClient discoveryClient = getInjector().getInstance(DiscoveryServiceClient.class);
    RemoteClientFactory remoteClientFactory = new RemoteClientFactory(discoveryClient, new DefaultInternalAuthenticator(new AuthenticationTestContext()));
    ArtifactLocalizerService artifactLocalizerService = new ArtifactLocalizerService(cConf, new ArtifactLocalizer(cConf, remoteClientFactory, (namespaceId, retryStrategy) -> {
        return new NoOpArtifactManager();
    }));
    // start the service
    artifactLocalizerService.startAndWait();
    return artifactLocalizerService;
}
Also used : NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Location(org.apache.twill.filesystem.Location) AccessException(io.cdap.cdap.api.security.AccessException) AppFabricTestBase(io.cdap.cdap.internal.app.services.http.AppFabricTestBase) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) NoOpInternalAuthenticator(io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator) InetAddress(java.net.InetAddress) Files(com.google.common.io.Files) AppJarHelper(io.cdap.cdap.common.test.AppJarHelper) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) After(org.junit.After) ArtifactInfo(io.cdap.cdap.api.artifact.ArtifactInfo) Locations(io.cdap.cdap.common.io.Locations) CloseableClassLoader(io.cdap.cdap.api.artifact.CloseableClassLoader) ClassRule(org.junit.ClassRule) Nullable(javax.annotation.Nullable) ArtifactManager(io.cdap.cdap.api.artifact.ArtifactManager) Before(org.junit.Before) AuthenticationTestContext(io.cdap.cdap.security.auth.context.AuthenticationTestContext) Test(org.junit.Test) IOException(java.io.IOException) LocationFactory(org.apache.twill.filesystem.LocationFactory) File(java.io.File) Id(io.cdap.cdap.common.id.Id) DefaultInternalAuthenticator(io.cdap.cdap.common.internal.remote.DefaultInternalAuthenticator) List(java.util.List) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TaskWorkerServiceTest(io.cdap.cdap.internal.app.worker.TaskWorkerServiceTest) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) Constants(io.cdap.cdap.common.conf.Constants) Assert(org.junit.Assert) DirUtils(io.cdap.cdap.common.utils.DirUtils) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) DefaultInternalAuthenticator(io.cdap.cdap.common.internal.remote.DefaultInternalAuthenticator) AuthenticationTestContext(io.cdap.cdap.security.auth.context.AuthenticationTestContext)

Example 2 with RemoteClientFactory

use of io.cdap.cdap.common.internal.remote.RemoteClientFactory in project cdap by caskdata.

the class MasterEnvironmentMain method doMain.

/**
 * The actual main method that get invoke through reflection from the {@link #main(String[])} method.
 */
@SuppressWarnings("unused")
public static void doMain(String[] args) throws Exception {
    CountDownLatch shutdownLatch = new CountDownLatch(1);
    try {
        // System wide setup
        Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
        // Intercept JUL loggers
        SLF4JBridgeHandler.removeHandlersForRootLogger();
        SLF4JBridgeHandler.install();
        EnvironmentOptions options = new EnvironmentOptions();
        String[] runnableArgs = OptionsParser.init(options, args, MasterEnvironmentMain.class.getSimpleName(), ProjectInfo.getVersion().toString(), System.out).toArray(new String[0]);
        String runnableClass = options.getRunnableClass();
        if (runnableClass == null) {
            throw new IllegalArgumentException("Missing runnable class name");
        }
        CConfiguration cConf = CConfiguration.create();
        SConfiguration sConf = SConfiguration.create();
        if (options.getExtraConfPath() != null) {
            cConf.addResource(new File(options.getExtraConfPath(), "cdap-site.xml").toURI().toURL());
            sConf.addResource(new File(options.getExtraConfPath(), "cdap-security.xml").toURI().toURL());
        }
        SecurityUtil.loginForMasterService(cConf);
        Configuration hConf = new Configuration();
        // Creates the master environment and load the MasterEnvironmentRunnable class from it.
        MasterEnvironment masterEnv = MasterEnvironments.setMasterEnvironment(MasterEnvironments.create(cConf, options.getEnvProvider()));
        MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
        masterEnv.initialize(context);
        try {
            Class<?> cls = masterEnv.getClass().getClassLoader().loadClass(runnableClass);
            if (!MasterEnvironmentRunnable.class.isAssignableFrom(cls)) {
                throw new IllegalArgumentException("Runnable class " + runnableClass + " is not an instance of " + MasterEnvironmentRunnable.class);
            }
            RemoteClientFactory remoteClientFactory = new RemoteClientFactory(masterEnv.getDiscoveryServiceClientSupplier().get(), getInternalAuthenticator(cConf), getRemoteAuthenticator(cConf));
            MasterEnvironmentRunnableContext runnableContext = new DefaultMasterEnvironmentRunnableContext(context.getLocationFactory(), remoteClientFactory);
            @SuppressWarnings("unchecked") MasterEnvironmentRunnable runnable = masterEnv.createRunnable(runnableContext, (Class<? extends MasterEnvironmentRunnable>) cls);
            AtomicBoolean completed = new AtomicBoolean();
            Runtime.getRuntime().addShutdownHook(new Thread(() -> {
                if (!completed.get()) {
                    runnable.stop();
                    Uninterruptibles.awaitUninterruptibly(shutdownLatch, 30, TimeUnit.SECONDS);
                }
                Optional.ofNullable(tokenManager).ifPresent(TokenManager::stopAndWait);
            }));
            runnable.run(runnableArgs);
            completed.set(true);
        } finally {
            masterEnv.destroy();
        }
    } catch (Exception e) {
        LOG.error("Failed to execute with arguments {}", Arrays.toString(args), e);
        throw e;
    } finally {
        shutdownLatch.countDown();
    }
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) MasterEnvironmentRunnable(io.cdap.cdap.master.spi.environment.MasterEnvironmentRunnable) CountDownLatch(java.util.concurrent.CountDownLatch) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MasterEnvironmentContext(io.cdap.cdap.master.spi.environment.MasterEnvironmentContext) MasterEnvironment(io.cdap.cdap.master.spi.environment.MasterEnvironment) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) UncaughtExceptionHandler(io.cdap.cdap.common.logging.common.UncaughtExceptionHandler) File(java.io.File) DefaultMasterEnvironmentRunnableContext(io.cdap.cdap.master.environment.DefaultMasterEnvironmentRunnableContext) MasterEnvironmentRunnableContext(io.cdap.cdap.master.spi.environment.MasterEnvironmentRunnableContext) DefaultMasterEnvironmentRunnableContext(io.cdap.cdap.master.environment.DefaultMasterEnvironmentRunnableContext)

Example 3 with RemoteClientFactory

use of io.cdap.cdap.common.internal.remote.RemoteClientFactory in project cdap by caskdata.

the class ArtifactCacheHttpHandlerInternal method fetchArtifact.

@GET
@Path("peers/{peer}/namespaces/{namespace-id}/artifacts/{artifact-name}/versions/{artifact-version}")
public void fetchArtifact(HttpRequest request, HttpResponder responder, @PathParam("peer") String peer, @PathParam("namespace-id") String namespaceId, @PathParam("artifact-name") String artifactName, @PathParam("artifact-version") String artifactVersion) throws Exception {
    ArtifactId artifactId = new ArtifactId(namespaceId, artifactName, artifactVersion);
    try {
        String endpoint = tetheringStore.getPeer(peer).getEndpoint();
        RemoteClientFactory factory = new RemoteClientFactory(new NoOpDiscoveryServiceClient(endpoint), new NoOpInternalAuthenticator(), remoteAuthenticator);
        HttpRequestConfig config = new DefaultHttpRequestConfig(true);
        RemoteClient remoteClient = factory.createRemoteClient("", config, Constants.Gateway.INTERNAL_API_VERSION_3);
        File artifactPath = cache.getArtifact(artifactId, peer, remoteClient);
        Location artifactLocation = Locations.toLocation(artifactPath);
        responder.sendContent(HttpResponseStatus.OK, new LocationBodyProducer(artifactLocation), new DefaultHttpHeaders().add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM));
    } catch (Exception ex) {
        if (ex instanceof HttpErrorStatusProvider) {
            HttpResponseStatus status = HttpResponseStatus.valueOf(((HttpErrorStatusProvider) ex).getStatusCode());
            responder.sendString(status, exceptionToJson(ex));
        } else {
            responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, exceptionToJson(ex));
        }
    }
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) DefaultHttpRequestConfig(io.cdap.cdap.common.http.DefaultHttpRequestConfig) HttpResponseStatus(io.netty.handler.codec.http.HttpResponseStatus) NoOpInternalAuthenticator(io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator) HttpErrorStatusProvider(io.cdap.cdap.api.common.HttpErrorStatusProvider) DefaultHttpRequestConfig(io.cdap.cdap.common.http.DefaultHttpRequestConfig) HttpRequestConfig(io.cdap.common.http.HttpRequestConfig) LocationBodyProducer(io.cdap.cdap.common.http.LocationBodyProducer) DefaultHttpHeaders(io.netty.handler.codec.http.DefaultHttpHeaders) RemoteClient(io.cdap.cdap.common.internal.remote.RemoteClient) File(java.io.File) Location(org.apache.twill.filesystem.Location) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Example 4 with RemoteClientFactory

use of io.cdap.cdap.common.internal.remote.RemoteClientFactory in project cdap by caskdata.

the class OAuthMacroEvaluatorTest method init.

@BeforeClass
public static void init() throws Exception {
    httpService = NettyHttpService.builder("OAuthTest").setHttpHandlers(new OAuthHandler(ImmutableMap.of(PROVIDER, ImmutableMap.of(CREDENTIAL_ID, new OAuthInfo("accessToken", "bearer"))))).build();
    httpService.start();
    InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService();
    String discoveryName = ServiceDiscoverable.getName(NamespaceId.SYSTEM.getNamespace(), Constants.PIPELINEID, ProgramType.SERVICE, Constants.STUDIO_SERVICE_NAME);
    discoveryService.register(new Discoverable(discoveryName, httpService.getBindAddress()));
    RemoteClientFactory remoteClientFactory = new RemoteClientFactory(discoveryService, new DefaultInternalAuthenticator(new AuthenticationTestContext()));
    serviceDiscoverer = new AbstractServiceDiscoverer(NamespaceId.DEFAULT.app("testapp").spark("testspark")) {

        @Override
        protected RemoteClientFactory getRemoteClientFactory() {
            return remoteClientFactory;
        }
    };
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) Discoverable(org.apache.twill.discovery.Discoverable) ServiceDiscoverable(io.cdap.cdap.common.service.ServiceDiscoverable) DefaultInternalAuthenticator(io.cdap.cdap.common.internal.remote.DefaultInternalAuthenticator) AuthenticationTestContext(io.cdap.cdap.security.auth.context.AuthenticationTestContext) AbstractServiceDiscoverer(io.cdap.cdap.app.services.AbstractServiceDiscoverer) InMemoryDiscoveryService(org.apache.twill.discovery.InMemoryDiscoveryService) BeforeClass(org.junit.BeforeClass)

Example 5 with RemoteClientFactory

use of io.cdap.cdap.common.internal.remote.RemoteClientFactory in project cdap by caskdata.

the class SystemAppTask method buildTaskSystemAppContext.

private SystemAppTaskContext buildTaskSystemAppContext(Injector injector, String systemAppNamespace, ArtifactId artifactId, ClassLoader artifactClassLoader) {
    PreferencesFetcher preferencesFetcher = injector.getInstance(PreferencesFetcher.class);
    PluginFinder pluginFinder = injector.getInstance(PluginFinder.class);
    SecureStore secureStore = injector.getInstance(SecureStore.class);
    ArtifactManagerFactory artifactManagerFactory = injector.getInstance(ArtifactManagerFactory.class);
    RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
    return new DefaultSystemAppTaskContext(injector.getInstance(CConfiguration.class), preferencesFetcher, pluginFinder, secureStore, systemAppNamespace, artifactId, artifactClassLoader, artifactManagerFactory, Constants.Service.TASK_WORKER, remoteClientFactory);
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) PluginFinder(io.cdap.cdap.internal.app.runtime.artifact.PluginFinder) ArtifactManagerFactory(io.cdap.cdap.internal.app.runtime.artifact.ArtifactManagerFactory) PreferencesFetcher(io.cdap.cdap.metadata.PreferencesFetcher) SecureStore(io.cdap.cdap.api.security.store.SecureStore) CConfiguration(io.cdap.cdap.common.conf.CConfiguration)

Aggregations

RemoteClientFactory (io.cdap.cdap.common.internal.remote.RemoteClientFactory)24 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)11 DefaultInternalAuthenticator (io.cdap.cdap.common.internal.remote.DefaultInternalAuthenticator)10 AuthenticationTestContext (io.cdap.cdap.security.auth.context.AuthenticationTestContext)10 Test (org.junit.Test)10 DiscoveryServiceClient (org.apache.twill.discovery.DiscoveryServiceClient)8 HttpRequestConfig (io.cdap.common.http.HttpRequestConfig)7 InMemoryDiscoveryService (org.apache.twill.discovery.InMemoryDiscoveryService)7 LocationFactory (org.apache.twill.filesystem.LocationFactory)7 RemoteClient (io.cdap.cdap.common.internal.remote.RemoteClient)6 BeforeClass (org.junit.BeforeClass)6 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)5 ArtifactRepository (io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository)5 File (java.io.File)4 IOException (java.io.IOException)4 RandomEndpointStrategy (io.cdap.cdap.common.discovery.RandomEndpointStrategy)3 Id (io.cdap.cdap.common.id.Id)3 NoOpInternalAuthenticator (io.cdap.cdap.common.internal.remote.NoOpInternalAuthenticator)3 DatasetService (io.cdap.cdap.data2.datafabric.dataset.service.DatasetService)3 DatasetOpExecutorService (io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService)3