Examples with ProgramId co.cask.cdap.proto.id.ProgramId used on opensource projects

Example 31 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LogHandlerTestRun method testPrevRunId.

private void testPrevRunId(String appId, String entityType, String entityId, String namespace, String format, List<String> suppress) throws Exception {
    ProgramId programId = new NamespaceId(namespace).app(appId).program(ProgramType.valueOfCategoryName(entityType), entityId);
    RunRecord runRecord = mockLogReader.getRunRecord(programId);
    int expectedEvents = 20;
    if (runRecord.getStatus() == ProgramRunStatus.RUNNING || runRecord.getStatus() == ProgramRunStatus.SUSPENDED) {
        expectedEvents = 30;
    }
    String prevRunIdUrl;
    if (suppress.isEmpty()) {
        prevRunIdUrl = String.format("apps/%s/%s/%s/runs/%s/logs/prev?format=%s&max=100", appId, entityType, entityId, runRecord.getPid(), format);
    } else {
        String fieldsToSuppress = getSuppressStr(suppress);
        prevRunIdUrl = String.format("apps/%s/%s/%s/runs/%s/logs/prev?format=%s&max=100&suppress=%s", appId, entityType, entityId, runRecord.getPid(), format, fieldsToSuppress);
    }
    HttpResponse response = doGet(getVersionedAPIPath(prevRunIdUrl, namespace));
    verifyLogs(response, entityId, format, true, false, true, expectedEvents, 20, suppress);
}

Example 32 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LogHandlerTestRun method testNativeMethodField.

// Verify the Json returned for logs has isNativeMethod set correctly
@Test
public void testNativeMethodField() throws Exception {
    ProgramId programId = new NamespaceId(MockLogReader.TEST_NAMESPACE).app("testTemplate1").program(ProgramType.valueOfCategoryName("workflows"), "testWorkflow1");
    RunRecord runRecord = mockLogReader.getRunRecord(programId);
    String logsUrl = String.format("apps/%s/%s/%s/runs/%s/logs/next?format=json", "testTemplate1", "workflows", "testWorkflow1", runRecord.getPid());
    HttpResponse response = doGet(getVersionedAPIPath(logsUrl, MockLogReader.TEST_NAMESPACE));
    Assert.assertEquals(HttpResponseStatus.OK.code(), response.getStatusLine().getStatusCode());
    String out = EntityUtils.toString(response.getEntity());
    List<LogDataOffset> logDataOffsetList = GSON.fromJson(out, LIST_LOGDATA_OFFSET_TYPE);
    Assert.assertEquals(logDataOffsetList.size(), 15);
    Assert.assertEquals(logDataOffsetList.get(0).getLog().getNativeMethod(), true);
    Assert.assertEquals(logDataOffsetList.get(1).getLog().getNativeMethod(), false);
    Assert.assertEquals(logDataOffsetList.get(2).getLog().getNativeMethod(), false);
}

Example 33 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class UserServiceEndpointStrategyTest method testStrategy.

@Test
public void testStrategy() throws Exception {
    ProgramId serviceId = new ApplicationId("n1", "a1").service("s1");
    String discoverableName = ServiceDiscoverable.getName(serviceId);
    List<Discoverable> candidates = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        candidates.add(new Discoverable(discoverableName, null, Bytes.toBytes(Integer.toString(i))));
    }
    SimpleServiceDiscovered serviceDiscovered = new SimpleServiceDiscovered(candidates);
    Map<String, Integer> routeToVersion = ImmutableMap.of("2", 100);
    Map<ProgramId, RouteConfig> routeMap = ImmutableMap.of(serviceId, new RouteConfig(routeToVersion));
    RouteStore configStore = new InMemoryRouteStore(routeMap);
    UserServiceEndpointStrategy strategy = new UserServiceEndpointStrategy(serviceDiscovered, configStore, serviceId);
    for (int i = 0; i < 1000; i++) {
        Discoverable picked = strategy.pick();
        Assert.assertEquals("2", Bytes.toString(picked.getPayload()));
    }
    // Switch config to choose version 3 always
    routeToVersion = ImmutableMap.of("3", 100);
    configStore.store(serviceId, new RouteConfig(routeToVersion));
    for (int i = 0; i < 1000; i++) {
        Discoverable picked = strategy.pick();
        Assert.assertEquals("3", Bytes.toString(picked.getPayload()));
    }
    // Switch config to choose verion 1 and 4 - 50% each
    routeToVersion = ImmutableMap.of("1", 50, "4", 50);
    configStore.store(serviceId, new RouteConfig(routeToVersion));
    Map<String, Integer> resultMap = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        Discoverable picked = strategy.pick();
        String version = Bytes.toString(picked.getPayload());
        if (resultMap.containsKey(version)) {
            resultMap.put(version, resultMap.get(version) + 1);
        } else {
            resultMap.put(version, 1);
        }
    }
    Assert.assertEquals(2, resultMap.size());
    double requestsToOne = resultMap.get("1");
    double requestsToTwo = resultMap.get("4");
    double requestRatio = requestsToOne / requestsToTwo;
    // Request Ratio should be close to 1.0 since we expect 50% of requests to go to each of these versions
    Assert.assertTrue(String.format("RequestRatio was %f and 1 got %f and 4 got %f", requestRatio, requestsToOne, requestsToTwo), requestRatio >= 0.7);
    Assert.assertTrue(String.format("RequestRatio was %f and 1 got %f and 4 got %f", requestRatio, requestsToOne, requestsToTwo), requestRatio <= 1.3);
    // Set the payload filter
    strategy = new UserServiceEndpointStrategy(serviceDiscovered, configStore, serviceId, null, "1");
    for (int i = 0; i < 1000; i++) {
        Discoverable picked = strategy.pick();
        Assert.assertEquals("1", Bytes.toString(picked.getPayload()));
    }
}

Example 34 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class AuditPublishersTest method testPublishingAccessLogs.

@Test
public void testPublishingAccessLogs() {
    String datasetName = "dummyDataset";
    String datasetName2 = "dummyDataset2";
    String appName = "dummyApp";
    String workerName = "dummyWorker";
    String workerName2 = "dummyWorker2";
    InMemoryAuditPublisher auditPublisher = new InMemoryAuditPublisher();
    ProgramId workerId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), appName, ProgramType.WORKER, workerName);
    DatasetId datasetId = NamespaceId.DEFAULT.dataset(datasetName);
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    List<AuditMessage> messages = auditPublisher.popMessages();
    // Since it is a READ_WRITE access, two messages are expected
    Assert.assertEquals(2, messages.size());
    // Same access so no message should be published
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(0, messages.size());
    // Different accesstype, hence a message should be published
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(1, messages.size());
    // Different dataset name, hence a message should be published
    datasetId = NamespaceId.DEFAULT.dataset(datasetName2);
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(2, messages.size());
    // Different worker name, hence a message should be published
    workerId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), appName, ProgramType.WORKER, workerName2);
    AuditPublishers.publishAccess(auditPublisher, datasetId, AccessType.READ_WRITE, workerId);
    messages = auditPublisher.popMessages();
    Assert.assertEquals(2, messages.size());
}

Example 35 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class LineageCollapser method collapseRelations.

/**
 * Collapse {@link Relation}s based on {@link CollapseType}
 * @param relations lineage relations
 * @param collapseTypes fields to collapse relations on
 * @return collapsed relations
 */
public static Set<CollapsedRelation> collapseRelations(Iterable<Relation> relations, Set<CollapseType> collapseTypes) {
    Set<CollapsedRelation> collapsedRelations = new HashSet<>();
    Multimap<CollapseKey, Relation> multimap = HashMultimap.create();
    for (Relation relation : relations) {
        multimap.put(getCollapseKey(relation, collapseTypes), relation);
    }
    LOG.trace("Collapsed relations: {}", multimap.asMap());
    for (Map.Entry<CollapseKey, Collection<Relation>> collapsedEntry : multimap.asMap().entrySet()) {
        NamespacedEntityId data = collapsedEntry.getKey().data;
        ProgramId program = collapsedEntry.getKey().program;
        Set<AccessType> accessTypes = new HashSet<>();
        Set<RunId> runs = new HashSet<>();
        Set<NamespacedEntityId> components = new HashSet<>();
        for (Relation relation : collapsedEntry.getValue()) {
            accessTypes.add(relation.getAccess());
            runs.add(relation.getRun());
            components.addAll(relation.getComponents());
        }
        collapsedRelations.add(toCollapsedRelation(data, program, accessTypes, runs, components));
    }
    return collapsedRelations;
}