fix(engine, java): reduce heap retention by releasing AST and detection state eagerly

engine/src/main/java/com/ibm/engine/detection/DetectionStore.java

@@ -32,8 +32,10 @@
 import com.ibm.engine.rule.IDetectionRule;
 import com.ibm.engine.rule.MethodDetectionRule;
 import com.ibm.engine.rule.Parameter;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -259,6 +261,26 @@ public void analyse(@Nonnull final T tree) {
         this.statusReporting.emitFinding();
     }
 
+    public void release() {
+        Deque<DetectionStore<R, T, S, P>> stack = new ArrayDeque<>();
+        stack.push(this);
+        List<DetectionStore<R, T, S, P>> order = new ArrayList<>();
+        while (!stack.isEmpty()) {
+            DetectionStore<R, T, S, P> node = stack.pop();
+            order.add(node);
+            for (List<DetectionStore<R, T, S, P>> childStores : node.children.values()) {
+                childStores.forEach(stack::push);
+            }
+        }
+        // Post-order: reverse so deepest descendants are cleared before their ancestors.
+        Collections.reverse(order);
+        for (DetectionStore<R, T, S, P> node : order) {
+            node.detectionValues.clear();
+            node.children.clear();
+            node.actionValue = null;
+        }
+    }
+
     @SuppressWarnings("java:S3776")
     public void onReceivingNewDetection(@Nonnull IDetection<T> detection) {
         if (detection instanceof MethodDetection<T> methodDetection) {
@@ -293,34 +315,32 @@ public void onReceivingNewDetection(@Nonnull IDetection<T> detection) {
 
         } else if (detection instanceof ValueDetection<?, T> valueDetection) {
             final DetectableParameter<T> detectableParameter = valueDetection.detectableParameter();
+            Optional<IValue<T>> emittedValue =
+                    valueDetection.toValue(valueDetection.detectableParameter().getiValueFactory());
 
             final Optional<Integer> positionMove = detectableParameter.getShouldBeMovedUnder();
             // Check if the parameter should be moved under
             if (positionMove.isPresent()) {
                 final int id = positionMove.get();
                 // Get the iValue to be detected and store it in a variable
-                valueDetection
-                        .toValue(valueDetection.detectableParameter().getiValueFactory())
-                        .ifPresent(
-                                iValue -> {
-                                    // Create a detection store with the given parameters
-                                    DetectionStore<R, T, S, P> detectionStore =
-                                            new DetectionStore<>(
-                                                    level + 1,
-                                                    detectionRule,
-                                                    scanContext,
-                                                    handler,
-                                                    statusReporting);
-                                    // Compute the detection values for the given id
-                                    addValue(detectionStore, id, iValue);
-                                    // Attach the detection store to the given id
-                                    this.attach(id, detectionStore);
-                                });
+                emittedValue.ifPresent(
+                        iValue -> {
+                            // Create a detection store with the given parameters
+                            DetectionStore<R, T, S, P> detectionStore =
+                                    new DetectionStore<>(
+                                            level + 1,
+                                            detectionRule,
+                                            scanContext,
+                                            handler,
+                                            statusReporting);
+                            // Compute the detection values for the given id
+                            addValue(detectionStore, id, iValue);
+                            // Attach the detection store to the given id
+                            this.attach(id, detectionStore);
+                        });
             } else {
-                valueDetection
-                        .toValue(valueDetection.detectableParameter().getiValueFactory())
-                        .ifPresent(
-                                iValue -> addValue(this, detectableParameter.getIndex(), iValue));
+                emittedValue.ifPresent(
+                        iValue -> addValue(this, detectableParameter.getIndex(), iValue));
             }
 
             // follow method parameter related detection rules
@@ -392,6 +412,7 @@ public void onDetectedDependingParameter(
     }
 
     public void onNewHookRegistration(@Nonnull IHook<R, T, S, P> hook) {
+        statusReporting.onDeferredHookRegistration();
         handler.subscribeToHookDetectionObservable(hook, this);
     }
 

engine/src/main/java/com/ibm/engine/executive/DetectionExecutive.java

@@ -30,15 +30,18 @@
 import java.util.Collection;
 import java.util.List;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public class DetectionExecutive<R, T, S, P>
         implements IStatusReporting<R, T, S, P>, IDomainEvent<Finding<R, T, S, P>> {
     @Nonnull private final List<IObserver<Finding<R, T, S, P>>> listeners = new ArrayList<>();
 
     @Nonnull private final DetectionStore<R, T, S, P> rootDetectionStore;
-    @Nonnull private final T tree;
+    @Nullable private T tree;
     private int expectedRuleVisits;
     private int visitedRules = 0;
+    private boolean deferredHookRegistered = false;
+    private boolean released = false;
 
     public DetectionExecutive(
             @Nonnull final T tree,
@@ -52,7 +55,11 @@ public DetectionExecutive(
     }
 
     public void start() {
-        this.rootDetectionStore.analyse(tree);
+        try {
+            this.rootDetectionStore.analyse(tree);
+        } finally {
+            this.tree = null;
+        }
     }
 
     @Override
@@ -65,12 +72,18 @@ public void emitFinding(@Nonnull final DetectionStore<R, T, S, P> rootDetectionS
         if (this.expectedRuleVisits != this.visitedRules) {
             return;
         }
-        getRootStoresWithValue(rootDetectionStore)
-                .forEach(
-                        store -> {
-                            final Finding<R, T, S, P> finding = new Finding<>(store);
-                            this.notify(finding);
-                        });
+        try {
+            getRootStoresWithValue(rootDetectionStore)
+                    .forEach(
+                            store -> {
+                                final Finding<R, T, S, P> finding = new Finding<>(store);
+                                this.notify(finding);
+                            });
+        } finally {
+            if (!deferredHookRegistered) {
+                releaseResources();
+            }
+        }
     }
 
     @Override
@@ -83,6 +96,23 @@ public void addAdditionalExpectedRuleVisits(int number) {
         this.expectedRuleVisits += number;
     }
 
+    @Override
+    public void onDeferredHookRegistration() {
+        this.deferredHookRegistered = true;
+    }
+
+    public boolean hasDeferredHooks() {
+        return deferredHookRegistered;
+    }
+
+    public boolean isReleased() {
+        return released;
+    }
+
+    public void releaseDeferredResources() {
+        releaseResources();
+    }
+
     @Nonnull
     private List<DetectionStore<R, T, S, P>> getRootStoresWithValue(
             @Nonnull DetectionStore<R, T, S, P> detectionStore) {
@@ -110,4 +140,16 @@ public void unsubscribe(@Nonnull IObserver<Finding<R, T, S, P>> listener) {
     public void notify(@Nonnull Finding<R, T, S, P> finding) {
         this.listeners.forEach(listener -> listener.update(finding));
     }
+
+    private void releaseResources() {
+        if (released) {
+            return;
+        }
+        released = true;
+        rootDetectionStore.release();
+        listeners.clear();
+        // Defensive null — start()'s finally already clears tree; retained here for safety if
+        // releaseResources() is ever called independently in future.
+        tree = null;
+    }
 }

engine/src/main/java/com/ibm/engine/executive/IStatusReporting.java

@@ -30,4 +30,35 @@ public interface IStatusReporting<R, T, S, P> {
     void incrementVisitedRules();
 
     void addAdditionalExpectedRuleVisits(int number);
+
+    /**
+     * Called when a detection store registers a hook whose invocation may fire after the current
+     * rule-visit cycle has completed.
+     *
+     * @apiNote This method is triggered by {@link
+     *     com.ibm.engine.detection.DetectionStore#onNewHookRegistration} the first time a deferred
+     *     hook is registered on a given scan node. It signals that at least one pending hook
+     *     invocation may still emit findings after {@code emitFinding()} would otherwise release
+     *     resources.
+     * @implSpec Implementations that manage a release lifecycle <em>must</em> set an internal flag
+     *     to suppress eager resource cleanup until all deferred hook invocations have completed.
+     *     For example:
+     *     <pre>{@code
+     * @Override
+     * public void onDeferredHookRegistration() {
+     *     this.deferredHookRegistered = true;
+     *     // emitFinding() will now skip releaseResources() and instead wait
+     *     // for an explicit releaseDeferredResources() call from the scan driver.
+     * }
+     * }</pre>
+     *     Failing to suppress the release will cause the detection-store tree to be cleared before
+     *     the pending hook findings are emitted, silently dropping detections.
+     * @implNote The default no-op is intentional. Language-module status reporters written before
+     *     this method was introduced do not track deferred-hook lifecycles and remain fully
+     *     source-compatible without any code changes.
+     * @since PR #417
+     */
+    default void onDeferredHookRegistration() {
+        // Optional lifecycle callback for deferred hook-based findings.
+    }
 }