Examples with Path com.yahoo.path.Path used on opensource projects

Example 21 with Path

use of com.yahoo.path.Path in project vespa by vespa-engine.

the class RankingExpressionWithTensorFlowTestCase method testImportingFromStoredExpressionsWithMacroOverridingConstant.

@Test
public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException {
    String rankProfile = "  rank-profile my_profile {\n" + "    macro Placeholder() {\n" + "      expression: tensor(d0[2],d1[784])(0.0)\n" + "    }\n" + "    macro mnist_softmax_saved_layer_Variable_read() {\n" + "      expression: tensor(d1[10],d2[784])(0.0)\n" + "    }\n" + "    first-phase {\n" + "      expression: tensorflow('mnist_softmax/saved')" + "    }\n" + "  }";
    String vespaExpressionWithoutConstant = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), mnist_softmax_saved_layer_Variable_read, f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))";
    RankProfileSearchFixture search = fixtureWith(rankProfile, new StoringApplicationPackage(applicationDir));
    search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
    assertNull("Constant overridden by macro is not added", search.search().getRankingConstants().get("mnist_softmax_saved_layer_Variable_read"));
    assertLargeConstant("mnist_softmax_saved_layer_Variable_1_read", search, Optional.of(10L));
    // At this point the expression is stored - copy application to another location which do not have a models dir
    Path storedApplicationDirectory = applicationDir.getParentPath().append("copy");
    try {
        storedApplicationDirectory.toFile().mkdirs();
        IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
        StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory);
        RankProfileSearchFixture searchFromStored = fixtureWith(rankProfile, storedApplication);
        searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile");
        assertNull("Constant overridden by macro is not added", searchFromStored.search().getRankingConstants().get("mnist_softmax_saved_layer_Variable_read"));
        assertLargeConstant("mnist_softmax_saved_layer_Variable_1_read", searchFromStored, Optional.of(10L));
    } finally {
        IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
    }
}

Example 22 with Path

use of com.yahoo.path.Path in project vespa by vespa-engine.

the class RankingExpressionWithTensorFlowTestCase method testImportingFromStoredExpressionsWithSmallConstants.

@Test
public void testImportingFromStoredExpressionsWithSmallConstants() throws IOException {
    final String expression = "join(join(reduce(join(join(join(tf_macro_mnist_saved_dnn_hidden2_add, reduce(constant(mnist_saved_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), tf_macro_mnist_saved_dnn_hidden2_add, f(a,b)(max(a,b))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))";
    final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))";
    final String macroExpression2 = "join(reduce(join(join(join(tf_macro_mnist_saved_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), tf_macro_mnist_saved_dnn_hidden1_add, f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b))";
    StoringApplicationPackage application = new StoringApplicationPackage(applicationDir);
    RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", "tensorflow('mnist/saved')", null, null, "input", application);
    search.assertFirstPhaseExpression(expression, "my_profile");
    assertSmallConstant("mnist_saved_dnn_hidden2_Const", TensorType.fromSpec("tensor(d2[1])"), search);
    search.assertMacro(macroExpression1, "tf_macro_mnist_saved_dnn_hidden1_add", "my_profile");
    search.assertMacro(macroExpression2, "tf_macro_mnist_saved_dnn_hidden2_add", "my_profile");
    // At this point the expression is stored - copy application to another location which do not have a models dir
    Path storedApplicationDirectory = applicationDir.getParentPath().append("copy");
    try {
        storedApplicationDirectory.toFile().mkdirs();
        IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
        StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory);
        RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[1],d1[784])(0.0)", "tensorflow('mnist/saved')", null, null, "input", storedApplication);
        searchFromStored.assertFirstPhaseExpression(expression, "my_profile");
        assertSmallConstant("mnist_saved_dnn_hidden2_Const", TensorType.fromSpec("tensor(d2[1])"), search);
        searchFromStored.assertMacro(macroExpression1, "tf_macro_mnist_saved_dnn_hidden1_add", "my_profile");
        searchFromStored.assertMacro(macroExpression2, "tf_macro_mnist_saved_dnn_hidden2_add", "my_profile");
    } finally {
        IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
    }
}

Example 23 with Path

use of com.yahoo.path.Path in project vespa by vespa-engine.

the class ZooKeeperClient method writeSearchDefinitions.

private void writeSearchDefinitions(ApplicationPackage app) throws IOException {
    Collection<NamedReader> sds = app.getSearchDefinitions();
    if (sds.isEmpty()) {
        return;
    }
    Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR);
    configCurator.createNode(zkPath.getAbsolute());
    // Ensures that ranking expressions and other files are also fed.
    writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
    for (NamedReader sd : sds) {
        String name = sd.getName();
        Reader reader = sd.getReader();
        String data = com.yahoo.io.IOUtils.readAll(reader);
        reader.close();
        configCurator.putData(zkPath.getAbsolute(), name, data);
    }
}

Example 24 with Path

use of com.yahoo.path.Path in project vespa by vespa-engine.

the class SessionFactoryImpl method loadSession.

@Override
public LocalSession loadSession(long sessionId) {
    File sessionDir = getSessionAppDir(sessionId);
    ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir);
    Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId));
    SessionZooKeeperClient sessionZKClient = new SessionZooKeeperClient(curator, configCurator, sessionIdPath, defRepo, serverId, nodeFlavors);
    SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, hostRegistry, superModelGenerationCounter);
    return new LocalSession(tenant, sessionId, sessionPreparer, context);
}

Example 25 with Path

use of com.yahoo.path.Path in project vespa by vespa-engine.

the class Tenants method writeTenantPath.

/**
 * Writes the path of the given tenant into ZooKeeper, for watchers to react on
 *
 * @param name name of the tenant
 */
private synchronized void writeTenantPath(TenantName name) {
    Path tenantPath = getTenantPath(name);
    curator.createAtomically(tenantPath, tenantPath.append(Tenant.SESSIONS), tenantPath.append(Tenant.APPLICATIONS));
}