Examples with Dependency org.apache.zeppelin.dep.Dependency used on opensource projects

Example 1 with Dependency

use of org.apache.zeppelin.dep.Dependency in project zeppelin by apache.

the class InterpreterSettingManager method loadInterpreterDependencies.

private void loadInterpreterDependencies(final InterpreterSetting setting) {
    setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
    setting.setErrorReason(null);
    interpreterSettings.put(setting.getId(), setting);
    synchronized (interpreterSettings) {
        final Thread t = new Thread() {

            public void run() {
                try {
                    // dependencies to prevent library conflict
                    File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + setting.getId());
                    if (localRepoDir.exists()) {
                        try {
                            FileUtils.forceDelete(localRepoDir);
                        } catch (FileNotFoundException e) {
                            logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
                        }
                    }
                    // load dependencies
                    List<Dependency> deps = setting.getDependencies();
                    if (deps != null) {
                        for (Dependency d : deps) {
                            File destDir = new File(zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
                            if (d.getExclusions() != null) {
                                dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), new File(destDir, setting.getId()));
                            } else {
                                dependencyResolver.load(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
                            }
                        }
                    }
                    setting.setStatus(InterpreterSetting.Status.READY);
                    setting.setErrorReason(null);
                } catch (Exception e) {
                    logger.error(String.format("Error while downloading repos for interpreter group : %s," + " go to interpreter setting page click on edit and save it again to make " + "this interpreter work properly. : %s", setting.getGroup(), e.getLocalizedMessage()), e);
                    setting.setErrorReason(e.getLocalizedMessage());
                    setting.setStatus(InterpreterSetting.Status.ERROR);
                } finally {
                    interpreterSettings.put(setting.getId(), setting);
                }
            }
        };
        t.start();
    }
}

Example 2 with Dependency

use of org.apache.zeppelin.dep.Dependency in project zeppelin by apache.

the class InterpreterSettingManager method copyDependenciesFromLocalPath.

/**
   * Overwrite dependency jar under local-repo/{interpreterId}
   * if jar file in original path is changed
   */
private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
    setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
    interpreterSettings.put(setting.getId(), setting);
    synchronized (interpreterSettings) {
        final Thread t = new Thread() {

            public void run() {
                try {
                    List<Dependency> deps = setting.getDependencies();
                    if (deps != null) {
                        for (Dependency d : deps) {
                            File destDir = new File(zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
                            int numSplits = d.getGroupArtifactVersion().split(":").length;
                            if (!(numSplits >= 3 && numSplits <= 6)) {
                                dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
                            }
                        }
                    }
                    setting.setStatus(InterpreterSetting.Status.READY);
                } catch (Exception e) {
                    logger.error(String.format("Error while copying deps for interpreter group : %s," + " go to interpreter setting page click on edit and save it again to make " + "this interpreter work properly.", setting.getGroup()), e);
                    setting.setErrorReason(e.getLocalizedMessage());
                    setting.setStatus(InterpreterSetting.Status.ERROR);
                } finally {
                    interpreterSettings.put(setting.getId(), setting);
                }
            }
        };
        t.start();
    }
}

Example 3 with Dependency

use of org.apache.zeppelin.dep.Dependency in project zeppelin by apache.

the class InterpreterSettingManager method add.

/**
   * @param group InterpreterSetting reference name
   */
public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos, List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> interpreterProperties, String path, InterpreterRunner runner) {
    Preconditions.checkNotNull(group, "name should not be null");
    Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
    Preconditions.checkNotNull(dependencies, "dependencies should not be null");
    Preconditions.checkNotNull(option, "option should not be null");
    Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
    InterpreterSetting interpreterSetting;
    synchronized (interpreterSettingsRef) {
        if (interpreterSettingsRef.containsKey(group)) {
            interpreterSetting = interpreterSettingsRef.get(group);
            // Append InterpreterInfo
            List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
            boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
            for (InterpreterInfo interpreterInfo : interpreterInfos) {
                if (!infos.contains(interpreterInfo)) {
                    if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
                        hasDefaultInterpreter = true;
                        infos.add(0, interpreterInfo);
                    } else {
                        infos.add(interpreterInfo);
                    }
                }
            }
            // Append dependencies
            List<Dependency> dependencyList = interpreterSetting.getDependencies();
            for (Dependency dependency : dependencies) {
                if (!dependencyList.contains(dependency)) {
                    dependencyList.add(dependency);
                }
            }
            // Append properties
            Map<String, InterpreterProperty> properties = (Map<String, InterpreterProperty>) interpreterSetting.getProperties();
            for (String key : interpreterProperties.keySet()) {
                if (!properties.containsKey(key)) {
                    properties.put(key, interpreterProperties.get(key));
                }
            }
        } else {
            interpreterSetting = new InterpreterSetting(group, null, interpreterInfos, interpreterProperties, dependencies, option, path, runner);
            interpreterSettingsRef.put(group, interpreterSetting);
        }
    }
    if (dependencies.size() > 0) {
        loadInterpreterDependencies(interpreterSetting);
    }
    interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory);
    return interpreterSetting;
}

Example 4 with Dependency

use of org.apache.zeppelin.dep.Dependency in project zeppelin by apache.

the class InterpreterFactoryTest method testRestartInterpreterInScopedMode.

/**
   * 2 users' interpreters in scoped mode. Each user has one session. Restarting user1's interpreter
   * won't affect user2's interpreter
   * @throws Exception
   */
@Test
public void testRestartInterpreterInScopedMode() throws Exception {
    interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
    ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
    interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
    interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(), Maps.<String, InterpreterProperty>newHashMap(), "mock1", null);
    Properties intp1Properties = new Properties();
    intp1Properties.put("PROPERTY_1", "VALUE_1");
    intp1Properties.put("property_2", "value_2");
    interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
    factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
    List<InterpreterSetting> all = interpreterSettingManager.get();
    InterpreterSetting mock1Setting = null;
    for (InterpreterSetting setting : all) {
        if (setting.getName().equals("mock1")) {
            mock1Setting = setting;
            break;
        }
    }
    mock1Setting.getOption().setPerUser("scoped");
    mock1Setting.getOption().setPerNote("shared");
    // set remote as false so that we won't create new remote interpreter process
    mock1Setting.getOption().setRemote(false);
    mock1Setting.getOption().setHost("localhost");
    mock1Setting.getOption().setPort(2222);
    InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user1", "sharedProcess");
    factory.createInterpretersForNote(mock1Setting, "user1", "sharedProcess", "user1");
    factory.createInterpretersForNote(mock1Setting, "user2", "sharedProcess", "user2");
    LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter) interpreterGroup.get("user1").get(0);
    interpreter1.open();
    LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter) interpreterGroup.get("user2").get(0);
    interpreter2.open();
    mock1Setting.closeAndRemoveInterpreterGroup("sharedProcess", "user1");
    assertFalse(interpreter1.isOpen());
    assertTrue(interpreter2.isOpen());
}

Example 5 with Dependency

use of org.apache.zeppelin.dep.Dependency in project zeppelin by apache.

the class InterpreterSettingTest method perUserIsolatedModeCloseAndRemoveInterpreterGroupTest.

@Test
public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() {
    InterpreterOption interpreterOption = new InterpreterOption();
    interpreterOption.setPerUser(InterpreterOption.ISOLATED);
    InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
    interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {

        @Override
        public InterpreterGroup createInterpreterGroup(String interpreterGroupId, InterpreterOption option) {
            return new InterpreterGroup(interpreterGroupId);
        }
    });
    Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
    List<Interpreter> interpreterList1 = new ArrayList<>();
    interpreterList1.add(mockInterpreter1);
    InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
    interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
    Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
    List<Interpreter> interpreterList2 = new ArrayList<>();
    interpreterList2.add(mockInterpreter2);
    interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
    interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
    assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
    assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
    assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").size());
    interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
    assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").size());
    assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
    interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
    assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}