diff --git a/shared-version.properties b/shared-version.properties index 19d155961a..ccd35e767d 100644 --- a/shared-version.properties +++ b/shared-version.properties @@ -1,2 +1,2 @@ -gradle.ext.blackDuckCommonVersion='67.0.20' +gradle.ext.blackDuckCommonVersion='68.0.0' gradle.ext.springBootVersion='2.7.12' diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/boot/product/ProductBoot.java b/src/main/java/com/blackduck/integration/detect/lifecycle/boot/product/ProductBoot.java index c907d525be..79939d22d2 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/boot/product/ProductBoot.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/boot/product/ProductBoot.java @@ -117,7 +117,6 @@ private BlackDuckRunData getBlackDuckRunData( BlackDuckServicesFactory blackDuckServicesFactory = blackDuckConnectivityResult.getBlackDuckServicesFactory(); setBlackDuckVersionLevel(blackDuckServicesFactory, blackDuckConnectivityResult); - boolean waitAtScanLevel = shouldWaitAtScanLevel(blackDuckConnectivityResult); // Fetch server detect properties settings Optional serverDetectProperties = fetchDetectPropertiesFromServer( @@ -126,7 +125,7 @@ private BlackDuckRunData getBlackDuckRunData( blackDuckServicesFactory.getBlackDuckApiClient() ); - return createBlackDuckRunDataBasedOnPhoneHomeDecision(blackDuckDecision, blackDuckServicesFactory, blackDuckConnectivityResult, waitAtScanLevel, serverDetectProperties); + return createBlackDuckRunDataBasedOnPhoneHomeDecision(blackDuckDecision, blackDuckServicesFactory, blackDuckConnectivityResult, serverDetectProperties); } else { if (productBootOptions.isIgnoreConnectionFailures()) { logger.info(String.format("Failed to connect to Black Duck SCA: %s", blackDuckConnectivityResult.getFailureReason())); @@ -148,7 +147,6 @@ private BlackDuckRunData createBlackDuckRunDataBasedOnPhoneHomeDecision( BlackDuckDecision blackDuckDecision, BlackDuckServicesFactory blackDuckServicesFactory, BlackDuckConnectivityResult blackDuckConnectivityResult, - boolean waitAtScanLevel, Optional serverDetectProperties ) { if (shouldUsePhoneHome(analyticsConfigurationService, blackDuckServicesFactory.getApiDiscovery(), blackDuckServicesFactory.getBlackDuckApiClient())) { @@ -160,7 +158,6 @@ private BlackDuckRunData createBlackDuckRunDataBasedOnPhoneHomeDecision( blackDuckServicesFactory, phoneHomeManager, blackDuckConnectivityResult, - waitAtScanLevel, serverDetectProperties ); } catch (IntegrationException e) { @@ -171,7 +168,7 @@ private BlackDuckRunData createBlackDuckRunDataBasedOnPhoneHomeDecision( } else { logger.debug("Skipping phone home due to Black Duck SCA global settings."); } - return BlackDuckRunData.onlineNoPhoneHome(blackDuckDecision.scanMode(), blackDuckServicesFactory, blackDuckConnectivityResult, waitAtScanLevel, serverDetectProperties); + return BlackDuckRunData.onlineNoPhoneHome(blackDuckDecision.scanMode(), blackDuckServicesFactory, blackDuckConnectivityResult, serverDetectProperties); } private Optional fetchDetectPropertiesFromServer( @@ -210,18 +207,4 @@ private boolean shouldUsePhoneHome(AnalyticsConfigurationService analyticsConfig return true; // Skip phone home will be applied at the library level. } } - - private boolean shouldWaitAtScanLevel(BlackDuckConnectivityResult blackDuckConnectivityResult) { - BlackDuckVersionParser parser = new BlackDuckVersionParser(); - Optional blackDuckServerVersion = parser.parse(blackDuckConnectivityResult.getContactedServerVersion()); - BlackDuckVersion minVersion = new BlackDuckVersion(2023, 1, 1); - - boolean waitAtScanLevel = false; - - if (blackDuckServerVersion.isPresent() && blackDuckServerVersion.get().isAtLeast(minVersion)) { - waitAtScanLevel = true; - } - - return waitAtScanLevel; - } } diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/data/BlackDuckRunData.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/data/BlackDuckRunData.java index c5eac9332f..185b4102c5 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/data/BlackDuckRunData.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/data/BlackDuckRunData.java @@ -16,7 +16,6 @@ public class BlackDuckRunData { private final BlackDuckServerConfig blackDuckServerConfig; private final BlackDuckServicesFactory blackDuckServicesFactory; private final BlackduckScanMode scanMode; - private final boolean waitAtScanLevel; private final Optional serverDetectProperties; private Optional blackDuckServerVersion; @@ -25,7 +24,6 @@ private BlackDuckRunData() { this.blackDuckServerConfig = null; this.blackDuckServicesFactory = null; this.scanMode = null; - this.waitAtScanLevel = false; this.serverDetectProperties = Optional.empty(); this.blackDuckServerVersion = Optional.empty(); } @@ -35,16 +33,14 @@ protected BlackDuckRunData( BlackDuckConnectivityResult blackDuckConnectivityResult, BlackDuckServicesFactory blackDuckServicesFactory, BlackduckScanMode scanMode, - boolean waitAtScanLevel, Optional serverDetectProperties ) { this.phoneHomeManager = phoneHomeManager; this.blackDuckServerConfig = blackDuckConnectivityResult != null ? blackDuckConnectivityResult.getBlackDuckServerConfig() : null; this.blackDuckServicesFactory = blackDuckServicesFactory; this.scanMode = scanMode; - this.waitAtScanLevel = waitAtScanLevel; - this.serverDetectProperties = serverDetectProperties; + this.serverDetectProperties = serverDetectProperties; determineBlackDuckServerVersion(blackDuckConnectivityResult); } @@ -73,20 +69,18 @@ public static BlackDuckRunData online( BlackDuckServicesFactory blackDuckServicesFactory, PhoneHomeManager phoneHomeManager, BlackDuckConnectivityResult blackDuckConnectivityResult, - boolean waitAtScanLevel, Optional serverDetectProperties ) { - return new BlackDuckRunData(phoneHomeManager, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, waitAtScanLevel, serverDetectProperties); + return new BlackDuckRunData(phoneHomeManager, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, serverDetectProperties); } public static BlackDuckRunData onlineNoPhoneHome( BlackduckScanMode scanMode, BlackDuckServicesFactory blackDuckServicesFactory, BlackDuckConnectivityResult blackDuckConnectivityResult, - boolean waitAtScanLevel, Optional serverDetectProperties ) { - return new BlackDuckRunData(null, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, waitAtScanLevel, serverDetectProperties); + return new BlackDuckRunData(null, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, serverDetectProperties); } public Boolean isNonPersistent() { @@ -96,10 +90,6 @@ public Boolean isNonPersistent() { public BlackduckScanMode getScanMode() { return scanMode; } - - public boolean shouldWaitAtScanLevel() { - return waitAtScanLevel; - } public Optional getBlackDuckServerVersion() { return blackDuckServerVersion; diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java index eb26518253..351d39caa5 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java @@ -710,7 +710,7 @@ private File createProtobufHeaderFile(String type, NameVersion projectNameVersio } } - public void uploadBdioEntries(BlackDuckRunData blackDuckRunData, UUID bdScanId) throws IntegrationException, IOException { + public void uploadBdioEntriesForRapidMode(BlackDuckRunData blackDuckRunData, UUID bdScanId) throws IntegrationException, IOException { // parse directory and upload all chunks File bdioDirectory = directoryManager.getBdioOutputDirectory(); @@ -1011,7 +1011,7 @@ public void attemptToGenerateComponentLocationAnalysisIfEnabled() throws Operati //Post actions //End post actions - public final BdioUploadResult uploadBdioIntelligentPersistent(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Long timeout, String scassScanId) throws OperationException { + public final BdioUploadResult uploadBdioForIntelligentPersistentMode(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Long timeout, String scassScanId) throws OperationException { return auditLog.namedPublic( "Upload Intelligent Persistent Bdio", () -> new IntelligentPersistentUploadOperation( @@ -1021,8 +1021,8 @@ public final BdioUploadResult uploadBdioIntelligentPersistent(BlackDuckRunData b ); } - public final CodeLocationWaitData calculateCodeLocationWaitData(List codeLocationCreationDatas) throws OperationException { - return auditLog.namedInternal("Calculate Code Location Wait Data", () -> new CodeLocationWaitCalculator().calculateWaitData(codeLocationCreationDatas)); + public final CodeLocationWaitData calculateCodeLocationWaitDataGivenNotificationRangeStart(List codeLocationCreationDatas, long codeLocationsUploadStartTime) throws OperationException { + return auditLog.namedInternal("Calculate Code Location Wait Data", () -> new CodeLocationWaitCalculator().calculateWaitData(codeLocationCreationDatas, codeLocationsUploadStartTime)); } public final void publishCodeLocationData(Set codeLocationData) throws OperationException { @@ -1297,13 +1297,6 @@ public ScanBatchRunner createScanBatchRunnerFromLocalInstall(File installDirecto }); } - public NotificationTaskRange createCodeLocationRange(BlackDuckRunData blackDuckRunData) throws OperationException { - return auditLog.namedInternal( - "Create Code Location Task Range", - () -> blackDuckRunData.getBlackDuckServicesFactory().createCodeLocationCreationService().calculateCodeLocationRange() - ); - } - public SignatureScanOuputResult signatureScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner) throws OperationException { return auditLog.namedPublic("Execute Signature Scan CLI", "SigScan", () -> new SignatureScanOperation().performScanActions(scanBatch, scanBatchRunner)); } @@ -1321,13 +1314,12 @@ public void publishSignatureScanReport(List report) thro } public SignatureScannerCodeLocationResult calculateWaitableSignatureScannerCodeLocations( - NotificationTaskRange notificationTaskRange, List reports ) throws OperationException { return auditLog.namedInternal( "Calculate Signature Scanner Waitable Code Locations", () -> new CalculateWaitableSignatureScanCodeLocations() - .calculateWaitableCodeLocations(notificationTaskRange, reports) + .calculateWaitableCodeLocations(reports) ); } diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IacScanStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IacScanStepRunner.java index f56ab34fe0..ebc6d57685 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IacScanStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IacScanStepRunner.java @@ -96,7 +96,7 @@ private IacScanReport performOnlineScan( try { File resultsFile = operationRunner.performIacScanScan(scanTarget, iacScanExe, count); String codeLocationName = operationRunner.createIacScanCodeLocationName(scanTarget, projectNameVersion); - String scanId = initiateScan(detectRunUuid, projectNameVersion, blackDuckRunData.getBlackDuckServicesFactory().createBdio2FileUploadService(), codeLocationName + String scanId = initiateIacScan(detectRunUuid, projectNameVersion, blackDuckRunData.getBlackDuckServicesFactory().createBdio2FileUploadService(), codeLocationName ); operationRunner.uploadIacScanResults(blackDuckRunData, resultsFile, scanId); return IacScanReport.SUCCESS_ONLINE(scanTarget, codeLocationName); @@ -121,7 +121,7 @@ public IacScanReport performOfflineScan(File scanTarget, File iacScanExe, int co //TODO- look into extracting scan initiation to another class //TODO- this should only be necessary if we didn't already upload BDIO during DETECTORS phase - private String initiateScan( + private String initiateIacScan( String detectRunUuid, NameVersion projectNameVersion, Bdio2FileUploadService bdio2FileUploadService, String codeLocationNameOverride ) diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java index 7f8408ff5f..2adb0e02f0 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java @@ -134,10 +134,15 @@ public void runOnline( logger.debug("Completed project and version actions."); logger.debug("Processing Detect Code Locations."); - - Set scanIdsToWaitFor = new HashSet<>(); + + // Code locations upload waiting mechanisms: + // Notifications based + long codeLocationsUploadStartTime = System.currentTimeMillis(); AtomicBoolean mustWaitAtBomSummaryLevel = new AtomicBoolean(false); + // ScanID based (/bom-status/{scanId} level) + Set scanIdsToWaitFor = new HashSet<>(); + CodeLocationAccumulator codeLocationAccumulator = new CodeLocationAccumulator(); if (bdioResult.isNotEmpty()) { @@ -146,8 +151,6 @@ public void runOnline( logger.debug("No BDIO results to upload. Skipping."); } - logger.debug("Completed Detect Code Location processing."); - stepHelper.runToolIfIncluded(DetectTool.SIGNATURE_SCAN, "Signature Scanner", () -> { SignatureScanStepRunner signatureScanStepRunner = new SignatureScanStepRunner(operationRunner, blackDuckRunData); SignatureScannerCodeLocationResult signatureScannerCodeLocationResult = signatureScanStepRunner.runSignatureScannerOnline( @@ -157,8 +160,9 @@ public void runOnline( scanIdsToWaitFor, gson ); - codeLocationAccumulator.addWaitableCodeLocations(signatureScannerCodeLocationResult.getWaitableCodeLocationData()); - codeLocationAccumulator.addNonWaitableCodeLocation(signatureScannerCodeLocationResult.getNonWaitableCodeLocationData()); + codeLocationAccumulator.addNonWaitableCodeLocations(signatureScannerCodeLocationResult.getWaitableCodeLocationData().getSuccessfulCodeLocationNames()); + codeLocationAccumulator.addNonWaitableCodeLocations(signatureScannerCodeLocationResult.getNonWaitableCodeLocationData()); + codeLocationAccumulator.incrementCodeLocationCountForTool(DetectTool.SIGNATURE_SCAN, 1); }); stepHelper.runToolIfIncluded(DetectTool.BINARY_SCAN, "Binary Scanner", () -> { @@ -176,7 +180,6 @@ public void runOnline( "Vulnerability Impact Analysis", () -> { runImpactAnalysisOnline(projectNameVersion, projectVersion, codeLocationAccumulator, blackDuckRunData.getBlackDuckServicesFactory()); - mustWaitAtBomSummaryLevel.set(true); }, operationRunner::publishImpactSuccess, operationRunner::publishImpactFailure @@ -186,10 +189,11 @@ public void runOnline( IacScanStepRunner iacScanStepRunner = new IacScanStepRunner(operationRunner); // IAC is not a supported correlated scan type, so pass null for correlation ID IacScanCodeLocationData iacScanCodeLocationData = iacScanStepRunner.runIacScanOnline(null, projectNameVersion, blackDuckRunData); - codeLocationAccumulator.addNonWaitableCodeLocation(iacScanCodeLocationData.getCodeLocationNames()); - mustWaitAtBomSummaryLevel.set(true); + codeLocationAccumulator.addNonWaitableCodeLocations(iacScanCodeLocationData.getCodeLocationNames()); }); + logger.debug("Completed Detect Code Location processing."); + if (correlatedScanningDecision.isEnabled()) { stepHelper.runAsGroup("Upload Correlated Scan Counts", OperationType.INTERNAL, () -> { uploadCorrelatedScanCounts(blackDuckRunData, codeLocationAccumulator); @@ -200,18 +204,15 @@ public void runOnline( stepHelper.runAsGroup("Wait for Results", OperationType.INTERNAL, () -> { // Calculate code locations. We do this even if we don't wait as we want to report code location data // in various reports. - CodeLocationResults codeLocationResults = calculateCodeLocations(codeLocationAccumulator); - - if (operationRunner.createBlackDuckPostOptions().shouldWaitForResults()) { - // Waiting at the scan level is more reliable, do that if the BD server is new enough. - if (blackDuckRunData.shouldWaitAtScanLevel()) { + CodeLocationResults codeLocationResults = calculateCodeLocations(codeLocationAccumulator, codeLocationsUploadStartTime); + + if (operationRunner.createBlackDuckPostOptions().shouldWaitForResults()) { + // Waiting at the scan level is more reliable, do that if the BD server is >= 2023.1.1 pollForBomScanCompletion(blackDuckRunData, projectVersion, scanIdsToWaitFor); - } - // If the BD server is older, or we can't detect its version, or if we have scans that we are - // not yet able to obtain the scanID for, use the original notification based waiting. - if (!blackDuckRunData.shouldWaitAtScanLevel() || mustWaitAtBomSummaryLevel.get()) { - waitForCodeLocations(codeLocationResults.getCodeLocationWaitData(), projectNameVersion, blackDuckRunData); + // If we have scans that we are not yet able to obtain the scanID for, use the original notification based waiting. + if (mustWaitAtBomSummaryLevel.get()) { + waitForWaitableCodeLocations(codeLocationResults.getCodeLocationWaitData(), projectNameVersion, blackDuckRunData); } } }); @@ -234,8 +235,9 @@ private void invokePackageManagerScanningWorkflow(NameVersion projectNameVersion scanId = commonScanResult.getScanId() == null ? null : commonScanResult.getScanId().toString(); if(commonScanResult.isPackageManagerScassPossible()) { scanIdsToWaitFor.add(scanId); + logger.debug("Added package manager scan {} to list of scanIds to wait for.", scanId); codeLocationAccumulator.addNonWaitableCodeLocation(commonScanResult.getCodeLocationName()); - codeLocationAccumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 1); + codeLocationAccumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 1); return; } } @@ -280,9 +282,11 @@ private void invokeBinaryScanningWorkflow( if (scanId.isPresent()) { scanIdsToWaitFor.add(scanId.get().toString()); + logger.debug("Added binary scan {} to list of scanIds to wait for.", scanId); } else { Optional> codeLocations = binaryScanStepRunner.getCodeLocations(); - + + // Waitable code Locations are only present if server version was too old for multipart binary upload (<2024.7.0) if (codeLocations.isPresent()) { codeLocationAccumulator.addWaitableCodeLocations(detectTool, codeLocations.get()); mustWaitAtBomSummaryLevel.set(true); @@ -306,10 +310,13 @@ private void invokeContainerScanningWorkflow( } Optional scanId = containerScanStepRunner.invokeContainerScanningWorkflow(); - scanId.ifPresent(uuid -> scanIdsToWaitFor.add(uuid.toString())); + scanId.ifPresent(uuid -> { + scanIdsToWaitFor.add(uuid.toString()); + logger.debug("Added container scan {} to list of scanIds to wait for.", uuid); + }); Set containerScanCodeLocations = new HashSet<>(); containerScanCodeLocations.add(containerScanStepRunner.getCodeLocationName()); - codeLocationAccumulator.addNonWaitableCodeLocation(containerScanCodeLocations); + codeLocationAccumulator.addNonWaitableCodeLocations(containerScanCodeLocations); } private void pollForBomScanCompletion(BlackDuckRunData blackDuckRunData, ProjectVersionWrapper projectVersion, @@ -329,15 +336,19 @@ private void pollForBomScanCompletion(BlackDuckRunData blackDuckRunData, Project } public void uploadBdio(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Set scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, Long timeout, String scassScanId) throws OperationException { - BdioUploadResult uploadResult = operationRunner.uploadBdioIntelligentPersistent(blackDuckRunData, bdioResult, timeout, scassScanId); + BdioUploadResult uploadResult = operationRunner.uploadBdioForIntelligentPersistentMode(blackDuckRunData, bdioResult, timeout, scassScanId); Optional> codeLocationCreationData = uploadResult.getUploadOutput(); - codeLocationCreationData.ifPresent(uploadBatchOutputCodeLocationCreationData -> codeLocationAccumulator.addWaitableCodeLocations( - DetectTool.DETECTOR, - uploadBatchOutputCodeLocationCreationData + codeLocationCreationData.ifPresent(uploadBatchOutputCodeLocationCreationData -> codeLocationAccumulator.addNonWaitableCodeLocations( + uploadBatchOutputCodeLocationCreationData.getOutput().getSuccessfulCodeLocationNames() )); + codeLocationAccumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 1); if (uploadResult.getUploadOutput().isPresent()) { for (UploadOutput result : uploadResult.getUploadOutput().get().getOutput()) { - result.getScanId().ifPresent((scanId) -> scanIdsToWaitFor.add(scanId)); + result.getScanId().ifPresent((scanId) -> { + scanIdsToWaitFor.add(scanId); + logger.debug("Added BDIO upload (prescass pkg mngr) scan ID to list of scanIds to wait for: {}", scanId); + } + ); } } } @@ -358,11 +369,11 @@ public void uploadCorrelatedScanCounts(BlackDuckRunData blackDuckRunData, CodeLo } } - public CodeLocationResults calculateCodeLocations(CodeLocationAccumulator codeLocationAccumulator) throws OperationException { //this is waiting.... + public CodeLocationResults calculateCodeLocations(CodeLocationAccumulator codeLocationAccumulator, long codeLocationsUploadStartTime) throws OperationException { logger.info(ReportConstants.RUN_SEPARATOR); Set allCodeLocationNames = new HashSet<>(codeLocationAccumulator.getNonWaitableCodeLocations()); - CodeLocationWaitData waitData = operationRunner.calculateCodeLocationWaitData(codeLocationAccumulator.getWaitableCodeLocations()); + CodeLocationWaitData waitData = operationRunner.calculateCodeLocationWaitDataGivenNotificationRangeStart(codeLocationAccumulator.getWaitableCodeLocations(), codeLocationsUploadStartTime); allCodeLocationNames.addAll(waitData.getCodeLocationNames()); Set allCodeLocationData = new HashSet<>(); @@ -405,10 +416,12 @@ private void checkPolicy(ProjectVersionView projectVersionView, BlackDuckRunData } } - public void waitForCodeLocations(CodeLocationWaitData codeLocationWaitData, NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData) + public void waitForWaitableCodeLocations(CodeLocationWaitData codeLocationWaitData, NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData) throws OperationException { logger.info("Checking to see if Detect should wait for bom tool calculations to finish."); - if (operationRunner.createBlackDuckPostOptions().shouldWaitForResults() && codeLocationWaitData.getExpectedNotificationCount() > 0) { + if (codeLocationWaitData.getExpectedNotificationCount() > 0) { + logger.debug("Will use old notifications based waiting for the following code locations: {}", codeLocationWaitData.getCodeLocationNames()); + logger.debug("Notifications after {} will be considered.", codeLocationWaitData.getNotificationRange().getStartDate()); operationRunner.waitForCodeLocations(blackDuckRunData, codeLocationWaitData, projectNameVersion); } } @@ -429,7 +442,7 @@ public void runImpactAnalysisOnline( ); operationRunner.mapImpactAnalysisCodeLocations(impactFile, uploadData, projectVersionWrapper, blackDuckServicesFactory); /* TODO: There is currently no mechanism within Black Duck for checking the completion status of an Impact Analysis code location. Waiting should happen here when such a mechanism exists. See HUB-25142. JM - 08/2020 */ - codeLocationAccumulator.addNonWaitableCodeLocation(uploadData.getOutput().getSuccessfulCodeLocationNames()); + codeLocationAccumulator.addNonWaitableCodeLocations(uploadData.getOutput().getSuccessfulCodeLocationNames()); } private Path generateImpactAnalysis(NameVersion projectNameVersion) throws OperationException { diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/RapidModeStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/RapidModeStepRunner.java index 2fd556eddf..6547aaa385 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/RapidModeStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/RapidModeStepRunner.java @@ -175,7 +175,7 @@ private void invokeBdbaRapidScan(BlackDuckRunData blackDuckRunData, String black rapidBdbaStepRunner.downloadAndExtractBdio(directoryManager); UUID bdScanId = operationRunner.initiateStatelessBdbaScan(blackDuckRunData); - operationRunner.uploadBdioEntries(blackDuckRunData, bdScanId); + operationRunner.uploadBdioEntriesForRapidMode(blackDuckRunData, bdScanId); // add this scan to the URLs to wait for parsedUrls.add(new HttpUrl(blackDuckUrl + String.format(RAPID_SCAN_ENDPOINT + "/" + bdScanId.toString()))); diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java index d2f20c1617..de841b6853 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java @@ -59,22 +59,21 @@ public SignatureScannerCodeLocationResult runSignatureScannerOnline(String detec List scanPaths = operationRunner.createScanPaths(projectNameVersion, dockerTargetData); ScanBatch scanBatch = operationRunner.createScanBatchOnline(detectRunUuid, scanPaths, projectNameVersion, dockerTargetData, blackDuckRunData, false); - NotificationTaskRange notificationTaskRange = operationRunner.createCodeLocationRange(blackDuckRunData); List reports; try { - reports = executeScan(scanBatch, scanBatchRunner, scanPaths, scanIdsToWaitFor, gson, blackDuckRunData.shouldWaitAtScanLevel(), true); + reports = executeScan(scanBatch, scanBatchRunner, scanPaths, scanIdsToWaitFor, gson, true); } catch (SocketException e) { if (!operationRunner.isCorrelationScanningEnabled("SIGNATURE")) { logger.warn("Initial Signature Scan failed due to connectivity issues. Retrying scan. Please allow the SCASS IPs to increase scanning performance."); scanBatch = operationRunner.createScanBatchOnline(detectRunUuid, scanPaths, projectNameVersion, dockerTargetData, blackDuckRunData, true); - reports = executeScan(scanBatch, scanBatchRunner, scanPaths, scanIdsToWaitFor, gson, blackDuckRunData.shouldWaitAtScanLevel(), true); + reports = executeScan(scanBatch, scanBatchRunner, scanPaths, scanIdsToWaitFor, gson, true); } else { reports = new ArrayList<>(); operationRunner.publishSignatureFailure("Correlation scanning is enabled. Please verify your SCASS configuration ad, as it is required for correlation scans to function properly."); } } - return operationRunner.calculateWaitableSignatureScannerCodeLocations(notificationTaskRange, reports); + return operationRunner.calculateWaitableSignatureScannerCodeLocations(reports); } public SignatureScanOuputResult runRapidSignatureScannerOnline(String detectRunUuid, BlackDuckRunData blackDuckRunData, NameVersion projectNameVersion, DockerTargetData dockerTargetData) @@ -99,15 +98,15 @@ public void runSignatureScannerOffline(String detectRunUuid, NameVersion project List scanPaths = operationRunner.createScanPaths(projectNameVersion, dockerTargetData); ScanBatch scanBatch = operationRunner.createScanBatchOffline(detectRunUuid, scanPaths, projectNameVersion, dockerTargetData); - executeScan(scanBatch, scanBatchRunner, scanPaths, null, null, false, false); + executeScan(scanBatch, scanBatchRunner, scanPaths, null, null,false); } - protected List executeScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner, List scanPaths, Set scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, boolean isOnline) throws OperationException, IOException { + protected List executeScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner, List scanPaths, Set scanIdsToWaitFor, Gson gson, boolean isOnline) throws OperationException, IOException { // Step 1: Run Scan CLI SignatureScanOuputResult scanOuputResult = operationRunner.signatureScan(scanBatch, scanBatchRunner); // Step 2: Check results and upload BDIO - Set failedScans = processEachScan(scanIdsToWaitFor, scanOuputResult, gson, shouldWaitAtScanLevel, scanBatch.isScassScan(), isOnline, scanBatch.isCsvArchive()); + Set failedScans = processEachScan(scanIdsToWaitFor, scanOuputResult, gson, scanBatch.isScassScan(), isOnline, scanBatch.isCsvArchive()); // Step 3: Report on results List reports = operationRunner.createSignatureScanReport(scanPaths, scanOuputResult.getScanBatchOutput().getOutputs(), failedScans); @@ -159,7 +158,7 @@ private ScanBatchRunnerUserResult findUserProvidedScanBatchRunner(Optional return ScanBatchRunnerUserResult.none(); } - private Set processEachScan(Set scanIdsToWaitFor, SignatureScanOuputResult signatureScanOutputResult, Gson gson, boolean shouldWaitAtScanLevel, boolean scassScan, boolean isOnline, boolean isCsvArchive) throws IOException { + private Set processEachScan(Set scanIdsToWaitFor, SignatureScanOuputResult signatureScanOutputResult, Gson gson, boolean scassScan, boolean isOnline, boolean isCsvArchive) throws IOException { List outputs = signatureScanOutputResult.getScanBatchOutput().getOutputs(); Set failedScans = new HashSet<>(); @@ -180,7 +179,7 @@ private Set processEachScan(Set scanIdsToWaitFor, SignatureScanO String scanOutputLocation = specificRunOutputDirectory.toString() + SignatureScanResult.OUTPUT_FILE_PATH; - processOnlineScan(scanIdsToWaitFor, gson, shouldWaitAtScanLevel, scassScan, failedScans, output, + processOnlineScan(scanIdsToWaitFor, gson, scassScan, failedScans, output, specificRunOutputDirectory, scanOutputLocation); } } @@ -188,14 +187,19 @@ private Set processEachScan(Set scanIdsToWaitFor, SignatureScanO return failedScans; } - private void processOnlineScan(Set scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, + private void processOnlineScan(Set scanIdsToWaitFor, Gson gson, boolean scassScan, Set failedScans, ScanCommandOutput output, File specificRunOutputDirectory, String scanOutputLocation) throws IOException, HttpHostConnectException, SocketException { - try { - Reader reader = Files.newBufferedReader(Paths.get(scanOutputLocation)); + SignatureScanResult result; + try (Reader reader = Files.newBufferedReader(Paths.get(scanOutputLocation))) { + result = gson.fromJson(reader, SignatureScanResult.class); + } catch (NoSuchFileException e) { + failedScans.add(output.getCodeLocationName()); + handleNoScanStatusFile(scassScan, scanOutputLocation); + return; + } - SignatureScanResult result = gson.fromJson(reader, SignatureScanResult.class); - + try { // This is a SCASS scan if we have an upload URL. We'll need to upload the BDIO. // If it is not a SCASS scan skip this section as the signature scanner already uploaded // the BDIO. @@ -207,12 +211,10 @@ private void processOnlineScan(Set scanIdsToWaitFor, Gson gson, boolean scassScanStepRunner.runScassScan(optionalBdio, result); } - if (shouldWaitAtScanLevel && scanIdsToWaitFor != null) { + if (scanIdsToWaitFor != null) { scanIdsToWaitFor.addAll(result.parseScanIds()); + logger.debug("Added the following signature scans to list of scanIds to wait for: {}." , result.parseScanIds()); } - } catch (NoSuchFileException e) { - failedScans.add(output.getCodeLocationName()); - handleNoScanStatusFile(scanIdsToWaitFor, shouldWaitAtScanLevel, scassScan, scanOutputLocation); } catch (IntegrationException e) { if (e.getCause() instanceof SocketException) { // The most likely cause of a failure like this is that the SCASS URLs are @@ -225,12 +227,12 @@ private void processOnlineScan(Set scanIdsToWaitFor, Gson gson, boolean } } - private void handleNoScanStatusFile(Set scanIdsToWaitFor, boolean shouldWaitAtScanLevel, boolean scassScan, + private void handleNoScanStatusFile(boolean scassScan, String scanOutputLocation) { if (scassScan) { String errorMessage = String.format("Unable to find scanOutput.json file at location: {}. Unable to upload BDIO to continue signature scan.", scanOutputLocation); operationRunner.publishSignatureFailure(errorMessage); - } else if (shouldWaitAtScanLevel && scanIdsToWaitFor != null) { + } else { logger.warn("Unable to find scanOutput.json file at location: " + scanOutputLocation + ". Will skip waiting for this signature scan."); } diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/PreScassBinaryScanStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/PreScassBinaryScanStepRunner.java index faead990ed..51a1e6954d 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/PreScassBinaryScanStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/PreScassBinaryScanStepRunner.java @@ -39,7 +39,7 @@ public UUID extractBinaryScanId(BinaryUploadStatus status) { @Override protected UUID performBlackduckInteractions(NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData, - Optional binaryScanFile) throws OperationException, IntegrationException { + Optional binaryScanFile) throws OperationException { if (isMultipartUploadPossible(blackDuckRunData)) { BinaryUploadStatus status = operationRunner.uploadBinaryScanFile(binaryScanFile.get(), projectNameVersion, blackDuckRunData); diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/ScassOrBdbaBinaryScanStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/ScassOrBdbaBinaryScanStepRunner.java index 7392ebe4f4..46e372e862 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/ScassOrBdbaBinaryScanStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/binary/ScassOrBdbaBinaryScanStepRunner.java @@ -23,7 +23,7 @@ public ScassOrBdbaBinaryScanStepRunner(OperationRunner operationRunner) { @Override public UUID performBlackduckInteractions(NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData, - Optional binaryScanFile) throws OperationException, IntegrationException { + Optional binaryScanFile) { try { CommonScanResult scanResult = commonScanStepRunner.performCommonScan( projectNameVersion, diff --git a/src/main/java/com/blackduck/integration/detect/tool/binaryscanner/BinaryUploadOperation.java b/src/main/java/com/blackduck/integration/detect/tool/binaryscanner/BinaryUploadOperation.java index 12a82464a9..38a6eea99c 100644 --- a/src/main/java/com/blackduck/integration/detect/tool/binaryscanner/BinaryUploadOperation.java +++ b/src/main/java/com/blackduck/integration/detect/tool/binaryscanner/BinaryUploadOperation.java @@ -107,6 +107,7 @@ public CodeLocationCreationData uploadLegacyBinaryScanFil NameVersion projectNameVersion ) throws DetectUserFriendlyException { + logger.debug("Using legacy binary scan upload method. This can be slow, consider upgrading to a newer version of Black Duck SCA to enable multipart uploading if possible."); String codeLocationName = codeLocationNameManager.createBinaryScanCodeLocationName( binaryScanFile, projectNameVersion.getName(), @@ -116,7 +117,7 @@ public CodeLocationCreationData uploadLegacyBinaryScanFil logger.info("Preparing to upload binary scan file: " + binaryScanFile.getAbsolutePath()); BinaryScan binaryScan = new BinaryScan(binaryScanFile, projectNameVersion.getName(), projectNameVersion.getVersion(), codeLocationName); BinaryScanBatch binaryScanBatch = new BinaryScanBatch(binaryScan); - CodeLocationCreationData codeLocationCreationData = binaryScanUploadService.uploadBinaryScan(binaryScanBatch); + CodeLocationCreationData codeLocationCreationData = binaryScanUploadService.uploadBinaryScanWithoutNotificationsQuery(binaryScanBatch); BinaryScanBatchOutput binaryScanBatchOutput = codeLocationCreationData.getOutput(); // The throwExceptionForError() in BinaryScanBatchOutput has a bug, so doing that work here diff --git a/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/ImpactAnalysisToolResult.java b/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/ImpactAnalysisToolResult.java deleted file mode 100644 index 20213607ec..0000000000 --- a/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/ImpactAnalysisToolResult.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.blackduck.integration.detect.tool.impactanalysis; - -import java.nio.file.Path; -import java.util.Collections; -import java.util.Set; - -import org.jetbrains.annotations.Nullable; - -import com.blackduck.integration.blackduck.codelocation.CodeLocationCreationData; -import com.blackduck.integration.blackduck.service.model.NotificationTaskRange; -import com.blackduck.integration.detect.tool.impactanalysis.service.ImpactAnalysisBatchOutput; - -public class ImpactAnalysisToolResult { - @Nullable - private final CodeLocationCreationData codeLocationCreationData; - @Nullable - private final Path impactAnalysisPath; - - public static ImpactAnalysisToolResult SUCCESS(CodeLocationCreationData codeLocationCreationData, Path impactAnalysisPath) { - return new ImpactAnalysisToolResult(codeLocationCreationData, impactAnalysisPath); - } - - public static ImpactAnalysisToolResult SUCCESS(Path impactAnalysisPath) { - return new ImpactAnalysisToolResult(null, impactAnalysisPath); - } - - public static ImpactAnalysisToolResult FAILURE() { - return new ImpactAnalysisToolResult(null, null); - } - - private ImpactAnalysisToolResult(@Nullable CodeLocationCreationData codeLocationCreationData, @Nullable Path impactAnalysisPath) { - this.codeLocationCreationData = codeLocationCreationData; - this.impactAnalysisPath = impactAnalysisPath; - } - - public boolean isSuccessful() { - if (null != codeLocationCreationData) { - return !codeLocationCreationData.getOutput().getSuccessfulCodeLocationNames().isEmpty(); - } else { - return null != impactAnalysisPath; - } - } - - public NotificationTaskRange getNotificationTaskRange() { - if (null == codeLocationCreationData) { - return null; - } - return codeLocationCreationData.getNotificationTaskRange(); - } - - public Set getCodeLocationNames() { - if (null == codeLocationCreationData) { - return Collections.emptySet(); - } - return codeLocationCreationData.getOutput().getSuccessfulCodeLocationNames(); - } - - @Nullable - public CodeLocationCreationData getCodeLocationCreationData() { - return codeLocationCreationData; - } - - @Nullable - public Path getImpactAnalysisPath() { - return impactAnalysisPath; - } -} diff --git a/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/service/ImpactAnalysisUploadService.java b/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/service/ImpactAnalysisUploadService.java index 88b82b9f53..80a58d3920 100644 --- a/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/service/ImpactAnalysisUploadService.java +++ b/src/main/java/com/blackduck/integration/detect/tool/impactanalysis/service/ImpactAnalysisUploadService.java @@ -1,15 +1,12 @@ package com.blackduck.integration.detect.tool.impactanalysis.service; -import java.util.Set; import com.blackduck.integration.blackduck.api.core.BlackDuckPath; import com.blackduck.integration.blackduck.api.manual.response.BlackDuckResponseResponse; import com.blackduck.integration.blackduck.codelocation.CodeLocationCreationData; import com.blackduck.integration.blackduck.codelocation.CodeLocationCreationService; import com.blackduck.integration.blackduck.service.BlackDuckServicesFactory; -import com.blackduck.integration.blackduck.service.model.NotificationTaskRange; import com.blackduck.integration.exception.IntegrationException; -import com.blackduck.integration.util.NameVersion; import com.blackduck.integration.util.NoThreadExecutorService; public class ImpactAnalysisUploadService { @@ -43,43 +40,13 @@ public ImpactAnalysisCodeLocationCreationRequest createUploadRequest(ImpactAnaly return new ImpactAnalysisCodeLocationCreationRequest(impactAnalysisBatchRunner, impactAnalysisBatch); } - public CodeLocationCreationData uploadImpactAnalysis(ImpactAnalysisCodeLocationCreationRequest uploadRequest) throws IntegrationException { - return codeLocationCreationService.createCodeLocations(uploadRequest); - } - public CodeLocationCreationData uploadImpactAnalysis(ImpactAnalysisBatch impactAnalysisBatch) throws IntegrationException { ImpactAnalysisCodeLocationCreationRequest uploadRequest = createUploadRequest(impactAnalysisBatch); - return codeLocationCreationService.createCodeLocations(uploadRequest); + return codeLocationCreationService.createCodeLocationsWithoutNotificationTaskRange(uploadRequest); } public CodeLocationCreationData uploadImpactAnalysis(ImpactAnalysis impactAnalysis) throws IntegrationException { ImpactAnalysisBatch impactAnalysisBatch = new ImpactAnalysisBatch(impactAnalysis); return uploadImpactAnalysis(impactAnalysisBatch); } - - public ImpactAnalysisBatchOutput uploadImpactAnalysisAndWait(ImpactAnalysisCodeLocationCreationRequest uploadRequest, long timeoutInSeconds) - throws InterruptedException, IntegrationException { - return codeLocationCreationService.createCodeLocationsAndWait(uploadRequest, timeoutInSeconds); - } - - public ImpactAnalysisBatchOutput uploadImpactAnalysisAndWait(ImpactAnalysisBatch impactAnalysisBatch, long timeoutInSeconds) throws InterruptedException, IntegrationException { - ImpactAnalysisCodeLocationCreationRequest uploadRequest = createUploadRequest(impactAnalysisBatch); - return uploadImpactAnalysisAndWait(uploadRequest, timeoutInSeconds); - } - - public ImpactAnalysisBatchOutput uploadImpactAnalysisAndWait(ImpactAnalysis impactAnalysis, long timeoutInSeconds) throws InterruptedException, IntegrationException { - ImpactAnalysisBatch impactAnalysisBatch = new ImpactAnalysisBatch(impactAnalysis); - return uploadImpactAnalysisAndWait(impactAnalysisBatch, timeoutInSeconds); - } - - public void waitForImpactAnalysisUpload( - NotificationTaskRange notificationTaskRange, - NameVersion projectAndVersion, - Set codeLocationNames, - int expectedNotificationCount, - long timeoutInSeconds - ) - throws InterruptedException, IntegrationException { - codeLocationCreationService.waitForCodeLocations(notificationTaskRange, projectAndVersion, codeLocationNames, expectedNotificationCount, timeoutInSeconds); - } } diff --git a/src/main/java/com/blackduck/integration/detect/tool/signaturescanner/operation/CalculateWaitableSignatureScanCodeLocations.java b/src/main/java/com/blackduck/integration/detect/tool/signaturescanner/operation/CalculateWaitableSignatureScanCodeLocations.java index 207fe7ecd6..a77ab1567d 100644 --- a/src/main/java/com/blackduck/integration/detect/tool/signaturescanner/operation/CalculateWaitableSignatureScanCodeLocations.java +++ b/src/main/java/com/blackduck/integration/detect/tool/signaturescanner/operation/CalculateWaitableSignatureScanCodeLocations.java @@ -5,7 +5,6 @@ import java.util.Set; import java.util.stream.Collectors; -import com.blackduck.integration.blackduck.service.model.NotificationTaskRange; import com.blackduck.integration.common.util.Bds; import com.blackduck.integration.detect.configuration.enumeration.DetectTool; import com.blackduck.integration.detect.tool.signaturescanner.SignatureScannerCodeLocationResult; @@ -13,15 +12,15 @@ import com.blackduck.integration.detect.workflow.blackduck.codelocation.WaitableCodeLocationData; public class CalculateWaitableSignatureScanCodeLocations { - public SignatureScannerCodeLocationResult calculateWaitableCodeLocations(NotificationTaskRange notificationTaskRange, List reports) { + public SignatureScannerCodeLocationResult calculateWaitableCodeLocations(List reports) { - WaitableCodeLocationData waitableCodeLocationData = calculateWaitable(notificationTaskRange, reports); + WaitableCodeLocationData waitableCodeLocationData = calculateWaitable(reports); Set nonWaitableNames = calculateNonWaitable(reports); return new SignatureScannerCodeLocationResult(waitableCodeLocationData, nonWaitableNames); } - private WaitableCodeLocationData calculateWaitable(NotificationTaskRange notificationTaskRange, List reports) { + private WaitableCodeLocationData calculateWaitable(List reports) { List waitableScans = Bds.of(reports) .filter(SignatureScannerReport::isSuccessful) .toList(); @@ -38,7 +37,7 @@ private WaitableCodeLocationData calculateWaitable(NotificationTaskRange notific .map(Optional::get) .collect(Collectors.toSet()); - return new WaitableCodeLocationData(DetectTool.SIGNATURE_SCAN, totalExpected, allNames, notificationTaskRange); + return new WaitableCodeLocationData(DetectTool.SIGNATURE_SCAN, totalExpected, allNames); } private Set calculateNonWaitable(List reports) { diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/bdio/IntelligentPersistentUploadOperation.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/bdio/IntelligentPersistentUploadOperation.java index f926e8e380..8b45910795 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/bdio/IntelligentPersistentUploadOperation.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/bdio/IntelligentPersistentUploadOperation.java @@ -18,6 +18,6 @@ public IntelligentPersistentUploadOperation(IntelligentPersistenceService intell @Override protected CodeLocationCreationData executeUpload(UploadBatch uploadBatch) throws IntegrationException { - return intelligentPersistenceService.uploadBdio(uploadBatch, timeout, Application.START_TIME); + return intelligentPersistenceService.uploadBdioWithoutNotificationsQuery(uploadBatch, timeout, Application.START_TIME); } } diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java index 98eaac8735..6a64c05126 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java @@ -19,8 +19,7 @@ public class CodeLocationAccumulator { public void addWaitableCodeLocations(DetectTool detectTool, CodeLocationCreationData> creationData) { addWaitableCodeLocations(new WaitableCodeLocationData(detectTool, creationData.getOutput().getExpectedNotificationCount(), - creationData.getOutput().getSuccessfulCodeLocationNames(), - creationData.getNotificationTaskRange() + creationData.getOutput().getSuccessfulCodeLocationNames() )); } @@ -28,7 +27,7 @@ public void addWaitableCodeLocations(WaitableCodeLocationData codeLocationData) waitableCodeLocationData.add(codeLocationData); } - public void addNonWaitableCodeLocation(Set names) { + public void addNonWaitableCodeLocations(Set names) { nonWaitableCodeLocations.addAll(names); } @@ -36,7 +35,11 @@ public void addNonWaitableCodeLocation(String name) { nonWaitableCodeLocations.add(name); } - public void incrementAdditionalCounts(DetectTool tool, int count) { + /** Increments the count of code locations for the given tool which will later be used by + * {@link com.blackduck.integration.detect.workflow.blackduck.integratedmatching.ScanCountsPayloadCreator} + * when correlated scanning is enabled. This method must be called whenever a code location is created for a tool + * that supports correlated scanning. */ + public void incrementCodeLocationCountForTool(DetectTool tool, int count) { additionalCountsByTool.put(tool, additionalCountsByTool.getOrDefault(tool, 0) + count); } diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java index c8d24caaf8..7f6262d51e 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java @@ -1,5 +1,8 @@ package com.blackduck.integration.detect.workflow.blackduck.codelocation; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.Date; import java.util.HashSet; import java.util.List; @@ -8,40 +11,24 @@ import com.blackduck.integration.blackduck.service.model.NotificationTaskRange; public class CodeLocationWaitCalculator { - public CodeLocationWaitData calculateWaitData(List codeLocationCreationDatas) { + public CodeLocationWaitData calculateWaitData(List codeLocationCreationDatas, long codeLocationsUploadStartTime) { int expectedNotificationCount = 0; - NotificationTaskRange notificationRange = null; + NotificationTaskRange notificationRange = getNotificationRangeGivenUploadStartTime(codeLocationsUploadStartTime); Set codeLocationNames = new HashSet<>(); for (WaitableCodeLocationData codeLocationCreationData : codeLocationCreationDatas) { expectedNotificationCount += codeLocationCreationData.getExpectedNotificationCount(); codeLocationNames.addAll(codeLocationCreationData.getSuccessfulCodeLocationNames()); - - if (null == notificationRange) { - notificationRange = codeLocationCreationData.getNotificationTaskRange(); - } else { - NotificationTaskRange rangeToAdd = codeLocationCreationData.getNotificationTaskRange(); - long earliestTaskTime = Math.min(notificationRange.getTaskStartTime(), rangeToAdd.getTaskStartTime()); - Date earliestStartDate = earliestDate(notificationRange.getStartDate(), rangeToAdd.getStartDate()); - Date latestEndDate = latestDate(notificationRange.getEndDate(), rangeToAdd.getEndDate()); - notificationRange = new NotificationTaskRange(earliestTaskTime, earliestStartDate, latestEndDate); - } } return new CodeLocationWaitData(notificationRange, codeLocationNames, expectedNotificationCount); } - private Date earliestDate(Date d1, Date d2) { - if (d1.before(d2)) { - return d1; - } - return d2; - } - - private Date latestDate(Date d1, Date d2) { - if (d1.after(d2)) { - return d1; - } - return d2; + public NotificationTaskRange getNotificationRangeGivenUploadStartTime(long codeLocationProcessingStartTime) { + LocalDateTime localStartTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(codeLocationProcessingStartTime), ZoneOffset.UTC); + LocalDateTime threeDaysLater = localStartTime.plusDays(3L); + Date startDate = Date.from(localStartTime.atZone(ZoneOffset.UTC).toInstant()); + Date endDate = Date.from(threeDaysLater.atZone(ZoneOffset.UTC).toInstant()); + return new NotificationTaskRange(0, startDate, endDate); } } diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/WaitableCodeLocationData.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/WaitableCodeLocationData.java index ebac28b9d6..09f150b9ab 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/WaitableCodeLocationData.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/WaitableCodeLocationData.java @@ -2,20 +2,17 @@ import java.util.Set; -import com.blackduck.integration.blackduck.service.model.NotificationTaskRange; import com.blackduck.integration.detect.configuration.enumeration.DetectTool; public class WaitableCodeLocationData { private final DetectTool detectTool; private final int expectedNotificationCount; private final Set successfulCodeLocationNames; - private final NotificationTaskRange notificationTaskRange; - public WaitableCodeLocationData(DetectTool detectTool, int expectedNotificationCount, Set successfulCodeLocationNames, NotificationTaskRange notificationTaskRange) { + public WaitableCodeLocationData(DetectTool detectTool, int expectedNotificationCount, Set successfulCodeLocationNames) { this.detectTool = detectTool; this.expectedNotificationCount = expectedNotificationCount; this.successfulCodeLocationNames = successfulCodeLocationNames; - this.notificationTaskRange = notificationTaskRange; } public DetectTool getDetectTool() { @@ -29,8 +26,4 @@ public int getExpectedNotificationCount() { public Set getSuccessfulCodeLocationNames() { return successfulCodeLocationNames; } - - public NotificationTaskRange getNotificationTaskRange() { - return notificationTaskRange; - } } diff --git a/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java b/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java index e947174251..58248d96f7 100644 --- a/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java +++ b/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java @@ -65,18 +65,17 @@ public void testRunSignatureScannerOnlineRetriesOnHttpHostConnectException() thr blackDuckRunData, false)).thenReturn(scanBatch); when(operationRunner.createScanBatchOnline(TEST_UUID, scanPaths, projectNameVersion, dockerTargetData, blackDuckRunData, true)).thenReturn(scanBatch); - when(blackDuckRunData.shouldWaitAtScanLevel()).thenReturn(true); SignatureScanStepRunner spyRunner = spy(signatureScanStepRunner); HttpHostConnectException httpException = createHttpHostConnectException(); doThrow(httpException).doReturn(Collections.emptyList()).when(spyRunner).executeScan(any(), any(), any(), any(), - any(), anyBoolean(), anyBoolean()); + any(), anyBoolean()); spyRunner.runSignatureScannerOnline(TEST_UUID, projectNameVersion, dockerTargetData, scanIdsToWaitFor, gson); - verify(spyRunner, times(2)).executeScan(any(), any(), any(), any(), any(), anyBoolean(), anyBoolean()); + verify(spyRunner, times(2)).executeScan(any(), any(), any(), any(), any(), anyBoolean()); } @Test @@ -89,16 +88,14 @@ public void testRunSignatureScannerOnlineExecutesOnceWithoutException() throws E when(operationRunner.createScanPaths(projectNameVersion, dockerTargetData)).thenReturn(scanPaths); when(operationRunner.createScanBatchOnline(TEST_UUID, scanPaths, projectNameVersion, dockerTargetData, blackDuckRunData, false)).thenReturn(scanBatch); - when(blackDuckRunData.shouldWaitAtScanLevel()).thenReturn(true); SignatureScanStepRunner spyRunner = spy(signatureScanStepRunner); - doReturn(Collections.emptyList()).when(spyRunner).executeScan(any(), any(), any(), any(), any(), anyBoolean(), - anyBoolean()); + doReturn(Collections.emptyList()).when(spyRunner).executeScan(any(), any(), any(), any(), any(), anyBoolean()); spyRunner.runSignatureScannerOnline(TEST_UUID, projectNameVersion, dockerTargetData, scanIdsToWaitFor, gson); - verify(spyRunner, times(1)).executeScan(any(), any(), any(), any(), any(), anyBoolean(), anyBoolean()); + verify(spyRunner, times(1)).executeScan(any(), any(), any(), any(), any(), anyBoolean()); } private HttpHostConnectException createHttpHostConnectException() { diff --git a/src/test/java/com/blackduck/integration/detect/tool/signaturescanner/CalculateWaitableSignatureScanCodeLocationsTest.java b/src/test/java/com/blackduck/integration/detect/tool/signaturescanner/CalculateWaitableSignatureScanCodeLocationsTest.java index 8b40df35da..f592af3d3e 100644 --- a/src/test/java/com/blackduck/integration/detect/tool/signaturescanner/CalculateWaitableSignatureScanCodeLocationsTest.java +++ b/src/test/java/com/blackduck/integration/detect/tool/signaturescanner/CalculateWaitableSignatureScanCodeLocationsTest.java @@ -57,8 +57,7 @@ public void combinationWaitsOnlyForSuccess() { } public SignatureScannerCodeLocationResult calculate(List reports) { - return new CalculateWaitableSignatureScanCodeLocations().calculateWaitableCodeLocations( - null, reports); + return new CalculateWaitableSignatureScanCodeLocations().calculateWaitableCodeLocations(reports); } public void assertWaited(SignatureScannerCodeLocationResult result, Set waited, int notificationCount) { diff --git a/src/test/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulatorTest.java b/src/test/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulatorTest.java index a0aeec6e6f..8095d7f0e7 100644 --- a/src/test/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulatorTest.java +++ b/src/test/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulatorTest.java @@ -36,8 +36,8 @@ void testAddMultipleNonWaitableCodeLocations() { Set names2 = new HashSet<>(Arrays.asList("location3", "location4")); String name3 = "location5"; - accumulator.addNonWaitableCodeLocation(names1); - accumulator.addNonWaitableCodeLocation(names2); + accumulator.addNonWaitableCodeLocations(names1); + accumulator.addNonWaitableCodeLocations(names2); accumulator.addNonWaitableCodeLocation(name3); assertEquals(5, accumulator.getNonWaitableCodeLocations().size()); @@ -47,27 +47,27 @@ void testAddMultipleNonWaitableCodeLocations() { } @Test - void testIncrementAdditionalCounts() { - accumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 5); + void testIncrementCodeLocationCountForTool() { + accumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 5); assertEquals(1, accumulator.getAdditionalCountsByTool().size()); assertEquals(5, accumulator.getAdditionalCountsByTool().get(DetectTool.DETECTOR)); } @Test - void testIncrementAdditionalCountsMultipleTimes() { - accumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 3); - accumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 7); + void testIncrementCodeLocationCountForToolMultipleTimes() { + accumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 3); + accumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 7); assertEquals(1, accumulator.getAdditionalCountsByTool().size()); assertEquals(10, accumulator.getAdditionalCountsByTool().get(DetectTool.DETECTOR)); } @Test - void testIncrementAdditionalCountsMultipleTools() { - accumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 3); - accumulator.incrementAdditionalCounts(DetectTool.SIGNATURE_SCAN, 5); - accumulator.incrementAdditionalCounts(DetectTool.BINARY_SCAN, 2); + void testIncrementCodeLocationCountForToolMultipleTools() { + accumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 3); + accumulator.incrementCodeLocationCountForTool(DetectTool.SIGNATURE_SCAN, 5); + accumulator.incrementCodeLocationCountForTool(DetectTool.BINARY_SCAN, 2); assertEquals(3, accumulator.getAdditionalCountsByTool().size()); assertEquals(3, accumulator.getAdditionalCountsByTool().get(DetectTool.DETECTOR)); @@ -76,8 +76,8 @@ void testIncrementAdditionalCountsMultipleTools() { } @Test - void testIncrementAdditionalCountsZero() { - accumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 0); + void testIncrementCodeLocationCountForToolZero() { + accumulator.incrementCodeLocationCountForTool(DetectTool.DETECTOR, 0); assertEquals(1, accumulator.getAdditionalCountsByTool().size()); assertEquals(0, accumulator.getAdditionalCountsByTool().get(DetectTool.DETECTOR));