Skip to content

changes for schema validation backend #5742

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kural-akto wants to merge 1 commit into
base: master
Choose a base branch
from
Open

changes for schema validation backend #5742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 157 additions & 0 deletions apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
Expand Down Expand Up @@ -622,4 +624,159 @@ public String getOpenApiSchema() {
return openApiSchema;
}

private Map<String, Object> schemaCoverageData;
private int skip;
private int limit;

public Map<String, Object> getSchemaCoverageData() {
return schemaCoverageData;
}

public void setSchemaCoverageData(Map<String, Object> schemaCoverageData) {
this.schemaCoverageData = schemaCoverageData;
}

public int getSkip() {
return skip;
}

public void setSkip(int skip) {
this.skip = skip;
}

public int getLimit() {
return limit;
}

public void setLimit(int limit) {
this.limit = limit;
}

private String validationSchema;
private boolean updateSuccess;

public String getValidationSchema() {
return validationSchema;
}

public void setValidationSchema(String validationSchema) {
this.validationSchema = validationSchema;
}

public boolean isUpdateSuccess() {
return updateSuccess;
}

public void setUpdateSuccess(boolean updateSuccess) {
this.updateSuccess = updateSuccess;
}

public String fetchSchemaCoverage() {
try {
List<SwaggerFileUpload> succeededUploads = FileUploadsDao.instance.getSwaggerMCollection()
.find(Filters.eq("uploadStatus", FileUpload.UploadStatus.SUCCEEDED.toString()))
.sort(Sorts.descending("uploadTs"))
.into(new ArrayList<>());

Set<Integer> uniqueCollectionIds = new HashSet<>();
for (SwaggerFileUpload upload : succeededUploads) {
uniqueCollectionIds.add(upload.getCollectionId());
}

Bson collectionFilter = Filters.and(
Filters.eq(ApiCollection._DEACTIVATED, false),
Filters.eq(ApiCollection.SAMPLE_COLLECTIONS_DROPPED, false)
);

long totalCollections = ApiCollectionsDao.instance.getMCollection()
.countDocuments(collectionFilter);

int collectionsCovered = uniqueCollectionIds.size();
double schemaCoverage = totalCollections > 0 ?
(double) collectionsCovered / totalCollections * 100 : 0;

int skipVal = skip > 0 ? skip : 0;
int limitVal = limit > 0 ? limit : 50;

com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper();
List<Map<String, Object>> dataList = succeededUploads
.stream()
.skip(skipVal)
.limit(limitVal)
.map(upload -> mapper.convertValue(upload, new com.fasterxml.jackson.core.type.TypeReference<Map<String, Object>>() {}))
.collect(Collectors.toList());

this.schemaCoverageData = new HashMap<>();
this.schemaCoverageData.put("data", dataList);
this.schemaCoverageData.put("totalCollections", totalCollections);
this.schemaCoverageData.put("collectionsCovered", collectionsCovered);
this.schemaCoverageData.put("schemaCoverage", Math.round(schemaCoverage * 100.0) / 100.0);
this.schemaCoverageData.put("total", collectionsCovered);

return SUCCESS.toUpperCase();
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, "ERROR while fetching schema coverage " + e);
addActionError("Error fetching schema coverage: " + e.getMessage());
return ERROR.toUpperCase();
}
}

public String addValidationSchema() {
try {
if (validationSchema == null || validationSchema.isEmpty()) {
addActionError("Validation schema is empty");
return ERROR.toUpperCase();
}

if (apiCollectionId <= 0) {
addActionError("Invalid API collection ID");
return ERROR.toUpperCase();
}

List<SwaggerFileUpload> fileUploads = FileUploadsDao.instance.getSwaggerMCollection()
.find(Filters.eq("collectionId", apiCollectionId))
.sort(Sorts.descending("uploadTs"))
.into(new ArrayList<>());

if (fileUploads.isEmpty()) {
addActionError("No file uploads found for collection ID: " + apiCollectionId);
return ERROR.toUpperCase();
}

SwaggerFileUpload latestUpload = fileUploads.get(0);
String swaggerFileId = latestUpload.getSwaggerFileId();

if (swaggerFileId == null || swaggerFileId.isEmpty()) {
addActionError("Invalid swagger file ID in file upload");
return ERROR.toUpperCase();
}

String compressedContent = GzipUtils.zipString(validationSchema);
int currentTimestamp = Context.now();

com.mongodb.client.result.UpdateResult updateResult = FilesDao.instance.getMCollection()
.updateOne(
Filters.eq("_id", new org.bson.types.ObjectId(swaggerFileId)),
Updates.combine(
Updates.set("compressedContent", compressedContent),
Updates.set("updatedAt", currentTimestamp)
)
);

if (updateResult.getModifiedCount() > 0) {
Map<String, Object> response = new HashMap<>();
response.put("success", true);
this.schemaCoverageData = response;
return SUCCESS.toUpperCase();
} else {
addActionError("File not found for ID: " + swaggerFileId);
return ERROR.toUpperCase();
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, "ERROR while adding validation schema " + e);
addActionError("Error adding validation schema: " + e.getMessage());
return ERROR.toUpperCase();
}
}

}
97 changes: 97 additions & 0 deletions apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.TimeRangeFilter;
import com.akto.util.enums.GlobalEnums.CONTEXT_SOURCE;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -85,6 +86,9 @@ public class SuspectSampleDataAction extends AbstractThreatDetectionAction {
@Getter @Setter List<String> hosts;
@Getter @Setter String latestApiOrigRegex;
@Getter @Setter Boolean sortBySeverity;
@Getter @Setter String filterId;
@Getter @Setter Map<String, Object> aggregatedData;
@Getter @Setter Map<String, Object> violationsOverTimeData;

// TODO: remove this, use API Executor.
private final CloseableHttpClient httpClient;
Expand Down Expand Up @@ -245,6 +249,99 @@ public String fetchSampleData() {
return SUCCESS.toUpperCase();
}

public String fetchAggregData() {
HttpPost post = new HttpPost(
String.format("%s/api/dashboard/aggregate_by_api_endpoint", this.getBackendUrl()));
post.addHeader("Authorization", "Bearer " + this.getApiToken());
post.addHeader("Content-Type", "application/json");
post.addHeader("x-context-source", Context.contextSource.get() != null ? Context.contextSource.get().toString() : "");

Map<String, Object> body = new HashMap<String, Object>() {
{
put("filterId", filterId);
put("skip", skip);
put("limit", limit > 0 ? limit : LIMIT);
put("startTs", startTimestamp);
put("endTs", endTimestamp);
}
};
String msg = objectMapper.valueToTree(body).toString();

StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON);
post.setEntity(requestEntity);

try (CloseableHttpResponse resp = this.httpClient.execute(post)) {
String responseBody = EntityUtils.toString(resp.getEntity());

if (resp.getStatusLine().getStatusCode() == 200) {
this.aggregatedData = objectMapper.readValue(responseBody, Map.class);
return SUCCESS.toUpperCase();
} else {
return ERROR.toUpperCase();
}
} catch (Exception e) {
e.printStackTrace();
return ERROR.toUpperCase();
}
}

public String fetchSchemaViolationsOverTime() {
HttpPost post = new HttpPost(
String.format("%s/api/dashboard/schema_violations_over_time", this.getBackendUrl()));
post.addHeader("Authorization", "Bearer " + this.getApiToken());
post.addHeader("Content-Type", "application/json");
post.addHeader("x-context-source", Context.contextSource.get() != null ? Context.contextSource.get().toString() : "");

Map<String, Object> body = new HashMap<String, Object>() {
{
put("filterId", filterId);
put("skip", skip);
put("limit", limit > 0 ? limit : LIMIT);
put("startTs", startTimestamp);
put("endTs", endTimestamp);
}
};
String msg = objectMapper.valueToTree(body).toString();

StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON);
post.setEntity(requestEntity);

try (CloseableHttpResponse resp = this.httpClient.execute(post)) {
String responseBody = EntityUtils.toString(resp.getEntity());

if (resp.getStatusLine().getStatusCode() == 200) {
Map<String, Object> backendResponse = objectMapper.readValue(responseBody, Map.class);

List<Map<String, Object>> eventsData = (List<Map<String, Object>>) backendResponse.get("data");
long daysBetween = ((Number) backendResponse.getOrDefault("daysBetween", 0)).longValue();

Map<String, Long> bucketCounts = new HashMap<>();
List<Integer> timestamps = new ArrayList<>();

if (eventsData != null) {
for (Map<String, Object> event : eventsData) {
long detectedAt = ((Number) event.get("detectedAt")).longValue();
timestamps.add((int) detectedAt);
}
}

bucketCounts = com.akto.util.TimePeriodUtils.groupByTimePeriod(timestamps, daysBetween);
List<List<Object>> timeSeries = com.akto.util.TimePeriodUtils.convertTimePeriodMapToTimeSeriesData(bucketCounts, daysBetween);

this.violationsOverTimeData = new HashMap<>();
this.violationsOverTimeData.put("data", timeSeries);
this.violationsOverTimeData.put("total", backendResponse.get("total"));

return SUCCESS.toUpperCase();
} else {
return ERROR.toUpperCase();
}
} catch (Exception e) {
e.printStackTrace();
return ERROR.toUpperCase();
}
}

public String fetchFilters() {
HttpPost post = new HttpPost(String.format("%s/api/dashboard/fetch_filters", this.getBackendUrl()));
post.addHeader("Authorization", "Bearer " + this.getApiToken());
Expand Down
Loading
Loading