diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java index 945b197b349..bf026371b56 100644 --- a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java @@ -61,8 +61,10 @@ import java.io.*; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -622,4 +624,159 @@ public String getOpenApiSchema() { return openApiSchema; } + private Map schemaCoverageData; + private int skip; + private int limit; + + public Map getSchemaCoverageData() { + return schemaCoverageData; + } + + public void setSchemaCoverageData(Map schemaCoverageData) { + this.schemaCoverageData = schemaCoverageData; + } + + public int getSkip() { + return skip; + } + + public void setSkip(int skip) { + this.skip = skip; + } + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + private String validationSchema; + private boolean updateSuccess; + + public String getValidationSchema() { + return validationSchema; + } + + public void setValidationSchema(String validationSchema) { + this.validationSchema = validationSchema; + } + + public boolean isUpdateSuccess() { + return updateSuccess; + } + + public void setUpdateSuccess(boolean updateSuccess) { + this.updateSuccess = updateSuccess; + } + + public String fetchSchemaCoverage() { + try { + List succeededUploads = FileUploadsDao.instance.getSwaggerMCollection() + .find(Filters.eq("uploadStatus", FileUpload.UploadStatus.SUCCEEDED.toString())) + .sort(Sorts.descending("uploadTs")) + .into(new ArrayList<>()); + + Set uniqueCollectionIds = new HashSet<>(); + for (SwaggerFileUpload upload : succeededUploads) { + uniqueCollectionIds.add(upload.getCollectionId()); + } + + Bson collectionFilter = Filters.and( + Filters.eq(ApiCollection._DEACTIVATED, false), + Filters.eq(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, false) + ); + + long totalCollections = ApiCollectionsDao.instance.getMCollection() + .countDocuments(collectionFilter); + + int collectionsCovered = uniqueCollectionIds.size(); + double schemaCoverage = totalCollections > 0 ? + (double) collectionsCovered / totalCollections * 100 : 0; + + int skipVal = skip > 0 ? skip : 0; + int limitVal = limit > 0 ? limit : 50; + + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + List> dataList = succeededUploads + .stream() + .skip(skipVal) + .limit(limitVal) + .map(upload -> mapper.convertValue(upload, new com.fasterxml.jackson.core.type.TypeReference>() {})) + .collect(Collectors.toList()); + + this.schemaCoverageData = new HashMap<>(); + this.schemaCoverageData.put("data", dataList); + this.schemaCoverageData.put("totalCollections", totalCollections); + this.schemaCoverageData.put("collectionsCovered", collectionsCovered); + this.schemaCoverageData.put("schemaCoverage", Math.round(schemaCoverage * 100.0) / 100.0); + this.schemaCoverageData.put("total", collectionsCovered); + + return SUCCESS.toUpperCase(); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "ERROR while fetching schema coverage " + e); + addActionError("Error fetching schema coverage: " + e.getMessage()); + return ERROR.toUpperCase(); + } + } + + public String addValidationSchema() { + try { + if (validationSchema == null || validationSchema.isEmpty()) { + addActionError("Validation schema is empty"); + return ERROR.toUpperCase(); + } + + if (apiCollectionId <= 0) { + addActionError("Invalid API collection ID"); + return ERROR.toUpperCase(); + } + + List fileUploads = FileUploadsDao.instance.getSwaggerMCollection() + .find(Filters.eq("collectionId", apiCollectionId)) + .sort(Sorts.descending("uploadTs")) + .into(new ArrayList<>()); + + if (fileUploads.isEmpty()) { + addActionError("No file uploads found for collection ID: " + apiCollectionId); + return ERROR.toUpperCase(); + } + + SwaggerFileUpload latestUpload = fileUploads.get(0); + String swaggerFileId = latestUpload.getSwaggerFileId(); + + if (swaggerFileId == null || swaggerFileId.isEmpty()) { + addActionError("Invalid swagger file ID in file upload"); + return ERROR.toUpperCase(); + } + + String compressedContent = GzipUtils.zipString(validationSchema); + int currentTimestamp = Context.now(); + + com.mongodb.client.result.UpdateResult updateResult = FilesDao.instance.getMCollection() + .updateOne( + Filters.eq("_id", new org.bson.types.ObjectId(swaggerFileId)), + Updates.combine( + Updates.set("compressedContent", compressedContent), + Updates.set("updatedAt", currentTimestamp) + ) + ); + + if (updateResult.getModifiedCount() > 0) { + Map response = new HashMap<>(); + response.put("success", true); + this.schemaCoverageData = response; + return SUCCESS.toUpperCase(); + } else { + addActionError("File not found for ID: " + swaggerFileId); + return ERROR.toUpperCase(); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "ERROR while adding validation schema " + e); + addActionError("Error adding validation schema: " + e.getMessage()); + return ERROR.toUpperCase(); + } + } + } diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java index 6176d0a7aa2..b991cde264b 100644 --- a/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java @@ -14,6 +14,7 @@ import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.TimeRangeFilter; import com.akto.util.enums.GlobalEnums.CONTEXT_SOURCE; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -85,6 +86,9 @@ public class SuspectSampleDataAction extends AbstractThreatDetectionAction { @Getter @Setter List hosts; @Getter @Setter String latestApiOrigRegex; @Getter @Setter Boolean sortBySeverity; + @Getter @Setter String filterId; + @Getter @Setter Map aggregatedData; + @Getter @Setter Map violationsOverTimeData; // TODO: remove this, use API Executor. private final CloseableHttpClient httpClient; @@ -245,6 +249,99 @@ public String fetchSampleData() { return SUCCESS.toUpperCase(); } + public String fetchAggregData() { + HttpPost post = new HttpPost( + String.format("%s/api/dashboard/aggregate_by_api_endpoint", this.getBackendUrl())); + post.addHeader("Authorization", "Bearer " + this.getApiToken()); + post.addHeader("Content-Type", "application/json"); + post.addHeader("x-context-source", Context.contextSource.get() != null ? Context.contextSource.get().toString() : ""); + + Map body = new HashMap() { + { + put("filterId", filterId); + put("skip", skip); + put("limit", limit > 0 ? limit : LIMIT); + put("startTs", startTimestamp); + put("endTs", endTimestamp); + } + }; + String msg = objectMapper.valueToTree(body).toString(); + + StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON); + post.setEntity(requestEntity); + + try (CloseableHttpResponse resp = this.httpClient.execute(post)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + if (resp.getStatusLine().getStatusCode() == 200) { + this.aggregatedData = objectMapper.readValue(responseBody, Map.class); + return SUCCESS.toUpperCase(); + } else { + return ERROR.toUpperCase(); + } + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + } + + public String fetchSchemaViolationsOverTime() { + HttpPost post = new HttpPost( + String.format("%s/api/dashboard/schema_violations_over_time", this.getBackendUrl())); + post.addHeader("Authorization", "Bearer " + this.getApiToken()); + post.addHeader("Content-Type", "application/json"); + post.addHeader("x-context-source", Context.contextSource.get() != null ? Context.contextSource.get().toString() : ""); + + Map body = new HashMap() { + { + put("filterId", filterId); + put("skip", skip); + put("limit", limit > 0 ? limit : LIMIT); + put("startTs", startTimestamp); + put("endTs", endTimestamp); + } + }; + String msg = objectMapper.valueToTree(body).toString(); + + StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON); + post.setEntity(requestEntity); + + try (CloseableHttpResponse resp = this.httpClient.execute(post)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + if (resp.getStatusLine().getStatusCode() == 200) { + Map backendResponse = objectMapper.readValue(responseBody, Map.class); + + List> eventsData = (List>) backendResponse.get("data"); + long daysBetween = ((Number) backendResponse.getOrDefault("daysBetween", 0)).longValue(); + + Map bucketCounts = new HashMap<>(); + List timestamps = new ArrayList<>(); + + if (eventsData != null) { + for (Map event : eventsData) { + long detectedAt = ((Number) event.get("detectedAt")).longValue(); + timestamps.add((int) detectedAt); + } + } + + bucketCounts = com.akto.util.TimePeriodUtils.groupByTimePeriod(timestamps, daysBetween); + List> timeSeries = com.akto.util.TimePeriodUtils.convertTimePeriodMapToTimeSeriesData(bucketCounts, daysBetween); + + this.violationsOverTimeData = new HashMap<>(); + this.violationsOverTimeData.put("data", timeSeries); + this.violationsOverTimeData.put("total", backendResponse.get("total")); + + return SUCCESS.toUpperCase(); + } else { + return ERROR.toUpperCase(); + } + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + } + public String fetchFilters() { HttpPost post = new HttpPost(String.format("%s/api/dashboard/fetch_filters", this.getBackendUrl())); post.addHeader("Authorization", "Bearer " + this.getApiToken()); diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index be16c444648..8bc61c57f13 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -2730,6 +2730,58 @@ + + + + + 403 + false + ^actionErrors.* + + + ACTIVE_ENDPOINTS + + + schemaCoverageData + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + ACTIVE_ENDPOINTS + + + 403 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + schemaCoverageData + + + 422 + false + ^actionErrors.* + + + @@ -11677,6 +11729,74 @@ + + + + + THREAT_PROTECTION + READ + + + THREAT_DETECTION + + + 403 + false + ^actionErrors.* + + + aggregatedData + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + + + + + + THREAT_PROTECTION + READ + + + THREAT_DETECTION + + + 403 + false + ^actionErrors.* + + + violationsOverTimeData + + + 422 + false + ^actionErrors.* + + + 403 + false + ^actionErrors.* + + + diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/router/DashboardRouter.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/router/DashboardRouter.java index 8805c51a74d..00053c2f0c9 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/router/DashboardRouter.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/router/DashboardRouter.java @@ -161,6 +161,86 @@ public Router setup(Vertx vertx) { ).ifPresent(s -> ctx.response().setStatusCode(200).end(s)); }); + router + .post("/aggregate_by_api_endpoint") + .blockingHandler(ctx -> { + String contextSource = getContextSourceHeader(ctx); + + RequestBody reqBody = ctx.body(); + String requestJson = reqBody.asString(); + + try { + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + Map req = mapper.readValue(requestJson, Map.class); + + String filterId = (String) req.get("filterId"); + if (filterId == null || filterId.isEmpty()) { + ctx.response().setStatusCode(400).end("{\"error\": \"filterId is required\"}"); + return; + } + + long startTs = ((Number) req.getOrDefault("startTs", 0L)).longValue(); + long endTs = ((Number) req.getOrDefault("endTs", 0L)).longValue(); + int skip = ((Number) req.getOrDefault("skip", 0)).intValue(); + int limit = ((Number) req.getOrDefault("limit", 50)).intValue(); + + String response = threatApiService.aggregateEventsByApiEndpoint( + ctx.get("accountId"), + filterId, + startTs, + endTs, + skip, + limit, + contextSource + ); + + ctx.response().setStatusCode(200).end(response); + } catch (Exception e) { + logger.error("Error in aggregate_by_api_endpoint: " + e.getMessage()); + ctx.response().setStatusCode(400).end("{\"error\": \"" + e.getMessage() + "\"}"); + } + }); + + router + .post("/schema_violations_over_time") + .blockingHandler(ctx -> { + String contextSource = getContextSourceHeader(ctx); + + RequestBody reqBody = ctx.body(); + String requestJson = reqBody.asString(); + + try { + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + Map req = mapper.readValue(requestJson, Map.class); + + String filterId = (String) req.get("filterId"); + if (filterId == null || filterId.isEmpty()) { + ctx.response().setStatusCode(400).end("{\"error\": \"filterId is required\"}"); + return; + } + + long startTs = ((Number) req.getOrDefault("startTs", 0L)).longValue(); + long endTs = ((Number) req.getOrDefault("endTs", 0L)).longValue(); + int skip = ((Number) req.getOrDefault("skip", 0)).intValue(); + int limit = ((Number) req.getOrDefault("limit", 50)).intValue(); + + String response = threatApiService.getSchemaViolationsOverTime( + ctx.get("accountId"), + filterId, + startTs, + endTs, + skip, + limit, + contextSource + ); + + ctx.response().setStatusCode(200).end(response); + } catch (Exception e) { + logger.error("Error in schema_violations_over_time: " + e.getMessage()); + ctx.response().setStatusCode(400).end("{\"error\": \"" + e.getMessage() + "\"}"); + } + }); + router .post("/delete_all_malicious_events") .blockingHandler(ctx -> { diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java index 1ce7f105a22..33401dd3f3c 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java @@ -10,6 +10,7 @@ import com.akto.threat.backend.dao.MaliciousEventDao; import com.akto.threat.backend.utils.ThreatUtils; import com.mongodb.client.MongoCursor; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; @@ -248,4 +249,222 @@ public ThreatSeverityWiseCountResponse getSeverityWiseCount( .build(); } + public String aggregateEventsByApiEndpoint( + String accountId, + String filterId, + long startTs, + long endTs, + int skip, + int limit, + String contextSource) { + + Document match = new Document(); + match.append("filterId", filterId); + + if (startTs > 0 || endTs > 0) { + Document timeRange = new Document(); + if (startTs > 0) { + timeRange.append("$gte", startTs); + } + if (endTs > 0) { + timeRange.append("$lte", endTs); + } + match.append("detectedAt", timeRange); + } + + Document contextFilter = ThreatUtils.buildSimpleContextFilter(contextSource, accountId); + if (!contextFilter.isEmpty()) { + match.putAll(contextFilter); + } + + List baseMatch = new ArrayList<>(); + if (!match.isEmpty()) { + baseMatch.add(new Document("$match", match)); + } + + List countPipeline = new ArrayList<>(baseMatch); + countPipeline.add(new Document("$count", "total")); + + Document countResult = maliciousEventDao.aggregateRaw(accountId, countPipeline).first(); + long total = countResult != null ? countResult.getInteger("total", 0) : 0; + + List dataPipeline = new ArrayList<>(baseMatch); + dataPipeline.add(new Document("$sort", new Document("detectedAt", -1))); + + dataPipeline.add( + new Document( + "$group", + new Document("_id", "$latestApiEndpoint") + .append("count", new Document("$sum", 1)) + .append("events", new Document("$push", new Document() + .append("id", "$_id") + .append("actor", "$actor") + .append("host", "$host") + .append("apiCollectionId", "$apiCollectionId") + .append("method", "$latestApiMethod") + .append("severity", "$severity") + .append("filterId", "$filterId") + .append("detectedAt", "$detectedAt") + .append("type", "$type") + .append("category", "$category") + .append("subCategory", "$subCategory") + .append("successfulExploit", "$successfulExploit") + .append("status", "$status") + .append("label", "$label") + .append("country", "$country") + .append("payload", "$payload") + .append("metadata", "$metadata") + .append("owaspCategories", "$owaspCategories") + .append("refId", "$refId") + .append("jiraTicketUrl", "$jiraTicketUrl"))))); + + dataPipeline.add(new Document("$skip", skip)); + dataPipeline.add(new Document("$limit", limit)); + + dataPipeline.add(new Document("$addFields", + new Document("events", new Document("$slice", Arrays.asList("$events", 10))))); + + List> apis = new ArrayList<>(); + + try (MongoCursor cursor = maliciousEventDao.aggregateRaw(accountId, dataPipeline).cursor()) { + while (cursor.hasNext()) { + Document doc = cursor.next(); + String endpoint = (String) doc.get("_id"); + long count = ((Number) doc.get("count")).longValue(); + List events = (List) doc.get("events"); + + Map apiData = new HashMap<>(); + apiData.put("endpoint", endpoint); + apiData.put("count", count); + + List> eventsList = new ArrayList<>(); + if (events != null) { + for (Document eventDoc : events) { + Map eventMap = new HashMap<>(eventDoc); + eventsList.add(eventMap); + } + } + apiData.put("events", eventsList); + apis.add(apiData); + } + } catch (Exception e) { + e.printStackTrace(); + } + + Map response = new HashMap<>(); + + List> dataArray = new ArrayList<>(); + for (Map api : apis) { + Map item = new HashMap<>(); + item.put("api", api.get("endpoint")); + item.put("events", api.get("events")); + item.put("count", api.get("count")); + dataArray.add(item); + } + + response.put("data", dataArray); + response.put("totalEndpoints", apis.size()); + response.put("total", total); + + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(response); + } catch (Exception e) { + e.printStackTrace(); + return "{\"error\": \"Failed to serialize response\"}"; + } + } + + @SuppressWarnings("unchecked") + public String getSchemaViolationsOverTime( + String accountId, + String filterId, + long startTs, + long endTs, + int skip, + int limit, + String contextSource) { + + Document match = new Document(); + match.append("filterId", filterId); + match.append("category", "SchemaConform"); + + if (startTs > 0 || endTs > 0) { + Document timeRange = new Document(); + if (startTs > 0) { + timeRange.append("$gte", startTs); + } + if (endTs > 0) { + timeRange.append("$lte", endTs); + } + match.append("detectedAt", timeRange); + } + + Document contextFilter = ThreatUtils.buildSimpleContextFilter(contextSource, accountId); + if (!contextFilter.isEmpty()) { + match.putAll(contextFilter); + } + + List baseMatch = new ArrayList<>(); + if (!match.isEmpty()) { + baseMatch.add(new Document("$match", match)); + } + + List countPipeline = new ArrayList<>(baseMatch); + countPipeline.add(new Document("$count", "total")); + + Document countResult = maliciousEventDao.aggregateRaw(accountId, countPipeline).first(); + long totalViolations = countResult != null ? countResult.getInteger("total", 0) : 0; + + if (totalViolations == 0) { + Map response = new HashMap<>(); + response.put("data", new ArrayList<>()); + response.put("total", 0); + + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(response); + } catch (Exception e) { + return "{\"error\": \"Failed to serialize response\"}"; + } + } + + long daysBetween = 0; + if (startTs > 0 && endTs > 0) { + daysBetween = (endTs - startTs) / 86400; + } + + List dataPipeline = new ArrayList<>(baseMatch); + dataPipeline.add(new Document("$sort", new Document("detectedAt", 1))); + dataPipeline.add(new Document("$skip", skip)); + dataPipeline.add(new Document("$limit", limit)); + + List> eventsData = new ArrayList<>(); + try (MongoCursor cursor = maliciousEventDao.aggregateRaw(accountId, dataPipeline).cursor()) { + while (cursor.hasNext()) { + Document doc = cursor.next(); + long detectedAt = ((Number) doc.get("detectedAt")).longValue(); + + Map item = new HashMap<>(); + item.put("detectedAt", detectedAt); + eventsData.add(item); + } + } catch (Exception e) { + e.printStackTrace(); + } + + Map response = new HashMap<>(); + response.put("data", eventsData); + response.put("total", totalViolations); + response.put("daysBetween", daysBetween); + + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(response); + } catch (Exception e) { + e.printStackTrace(); + return "{\"error\": \"Failed to serialize response\"}"; + } + } + }