Add Refset Maintenance Report listing inactive concepts in active simple-type refset members

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java

@@ -116,20 +116,15 @@ private static void populateKnownConcepts() {
 	public Collection <Concept> getAllConcepts() {
 		return concepts.values();
 	}
-
 
 	public void reset() {
-		LOGGER.info("Resetting Graph Loader - configuration reset");
-		setRecordPreviousState(false);
-
 		memoryWipe();
-
 		System.gc();
 		outputMemoryUsage();
 	}
 
 	public void memoryWipe() {
-		LOGGER.info("Resetting Graph Loader - memory wipe");
+		LOGGER.info("Resetting Graph Loader - memory wipe, retained configuration");
 		concepts = new HashMap<>();
 		mdrs = null;
 		descriptions = new HashMap<>();
@@ -717,7 +712,7 @@ public void loadConceptFile(InputStream is, Boolean isReleased) throws IOExcepti
 
 				//We might already have received some details about this concept
 				Concept c = getConcept(lineItems[IDX_ID]);
-				
+
 				//If moduleId is null, then this concept has no prior state
 				if (isRecordPreviousState()) {
 					if (isReleased == null) {

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java

@@ -8,6 +8,7 @@
 import java.util.stream.Stream;
 
 import org.ihtsdo.otf.rest.client.terminologyserver.pojo.*;
+import org.ihtsdo.otf.utils.SnomedUtilsBase;
 import org.ihtsdo.termserver.scripting.dao.ReportDataBroker;
 
 import org.apache.commons.lang.time.DurationFormatUtils;
@@ -138,6 +139,8 @@ public abstract class TermServerScript extends Script implements ScriptConstants
 
 	protected ReportDataBroker reportDataBroker;
 
+	private  Map<String, Map<String, Integer>> summaryCountsByCategory = new HashMap<>();
+
 	public static Gson gson;
 	static {
 		GsonBuilder gsonBuilder = new GsonBuilder();
@@ -611,7 +614,7 @@ protected void preInit() throws TermServerScriptException {
 		//like selfDetermining = true;
 	}
 
-	protected void runJob () throws TermServerScriptException {
+	public void runJob () throws TermServerScriptException {
 		throw new TermServerScriptException("Override this method in concrete class");
 	}
 
@@ -1844,27 +1847,49 @@ protected boolean inScope(Component c, boolean includeExpectedExtensionModules)
 		if (project.getKey().equals("MAIN")) {
 			return true;
 		}
-		//Do we have a default module id ie for a managed service project?
-		if (project.getMetadata() != null && project.getMetadata().getDefaultModuleId() != null) {
-			//We really need to be sure that expectedExtensionModules has been populated, 
-			//because CH and NO will have content in multiple modules
-			if (project.getMetadata().getExpectedExtensionModules() == null) {
-				if (allowMissingExpectedModules) {
-					return c.getModuleId().equals(project.getMetadata().getDefaultModuleId());
-				} else {
-					throw new IllegalArgumentException("Extension does not have expectedExtensionModules metadata populated.  Cannot continue.");
-				}
-			}
-			if (includeExpectedExtensionModules) {
-				return project.getMetadata().getExpectedExtensionModules().contains(c.getModuleId());
-			} else {
-				return c.getModuleId().equals(project.getMetadata().getDefaultModuleId());
-			}
+
+		List<String> inScopeModules = getInScopeModules(includeExpectedExtensionModules);
+		if (!inScopeModules.isEmpty()) {
+			return inScopeModules.contains(c.getModuleId());
 		} else if (moduleFilter != null) {
 			return moduleFilter.contains(c.getModuleId());
 		}
 		return true;
 	}
+
+	protected List<String> getInScopeModules(boolean includeExpectedExtensionModules) {
+		if (project.getMetadata() == null) {
+			return List.of();
+		}
+
+		var metadata = project.getMetadata();
+		var defaultModuleId = metadata.getDefaultModuleId();
+
+		if (defaultModuleId == null) {
+			return List.of();
+		}
+
+		var expectedModules = metadata.getExpectedExtensionModules();
+		if (expectedModules == null) {
+			if (!allowMissingExpectedModules) {
+				throw new IllegalArgumentException(
+						"Extension does not have expectedExtensionModules metadata populated. Cannot continue."
+				);
+			}
+			return List.of(defaultModuleId);
+		}
+
+		return includeExpectedExtensionModules
+				? expectedModules
+				: List.of(defaultModuleId);
+	}
+
+	protected Set<String> getInScopeNamespaces() {
+		return getInScopeModules(true).stream()
+				.map(SnomedUtilsBase::getNamespace)
+				.filter(Objects::nonNull)
+				.collect(Collectors.toUnmodifiableSet());
+	}
 
 	protected boolean isMS() {
 		//Do we have a default module id ie for a managed service project?
@@ -2361,7 +2386,7 @@ protected void shuffleDown(Task t, Concept c) throws TermServerScriptException {
 		List<RelationshipGroup> newGroups = new ArrayList<>();
 		for (RelationshipGroup group : c.getRelationshipGroups(CharacteristicType.STATED_RELATIONSHIP)) {
 			//Have we missed out the ungrouped group? fill in if so
-			if (group.isGrouped() && newGroups.size() == 0) {
+			if (group.isGrouped() && newGroups.isEmpty()) {
 				newGroups.add(new RelationshipGroup(UNGROUPED));
 			}
 			//Since we're working with the true concept relationships here, this will have
@@ -2375,7 +2400,7 @@ protected void shuffleDown(Task t, Concept c) throws TermServerScriptException {
 				for (Relationship moved : new ArrayList<>(group.getRelationships())) {
 					if (StringUtils.isEmpty(moved.getId())) {
 						Set<Relationship> existingInactives = c.getRelationships(moved, ActiveState.INACTIVE);
-						if (existingInactives.size() > 0) {
+						if (!existingInactives.isEmpty()) {
 							group.removeRelationship(moved);
 							c.removeRelationship(moved, true);  //It's OK to force removal, the axiom will still exist.
 							Relationship reuse = existingInactives.iterator().next();
@@ -2400,4 +2425,60 @@ public boolean isDryRun() {
 		return dryRun;
 	}
 
+	public void initialiseSummaryCount(String category, String item) {
+		summaryCountsByCategory
+				.computeIfAbsent(category, k -> new HashMap<>())
+				.putIfAbsent(item, 0);
+	}
+
+	public void incrementSummaryCount(String category, String summaryItem) {
+		incrementSummaryCount(category, summaryItem, 1);
+	}
+
+	public void incrementSummaryCount(String category, String summaryItem, int increment) {
+		//Increment the count for this summary item, in the appropriate category
+		Map<String, Integer> summaryCounts = summaryCountsByCategory.computeIfAbsent(category, k -> new HashMap<>());
+		summaryCounts.merge(summaryItem, increment, Integer::sum);
+	}
+
+	public enum SUMMARY_SORT_ORDER {
+		ALPHABETICAL,
+		COUNT
+	}
+
+	protected void reportSummaryCounts(int summaryTabIdx) throws TermServerScriptException {
+		reportSummaryCounts(summaryTabIdx, SUMMARY_SORT_ORDER.ALPHABETICAL);
+	}
+
+	protected void reportSummaryCounts(int summaryTabIdx, SUMMARY_SORT_ORDER sortOrder) throws TermServerScriptException {
+		report(summaryTabIdx, "");
+
+		// Work through each category (sorted alphabetically)
+		summaryCountsByCategory.keySet().stream()
+				.sorted()
+				.forEach(cat -> {
+					reportSafely(summaryTabIdx, cat);
+
+					Map<String, Integer> summaryCounts = summaryCountsByCategory.get(cat);
+					Stream<Map.Entry<String, Integer>> stream = summaryCounts.entrySet().stream();
+					if (sortOrder == SUMMARY_SORT_ORDER.COUNT) {
+						// Sort by count descending, then alphabetically for stability
+						stream = stream.sorted(
+								Comparator.comparing(Map.Entry<String, Integer>::getValue)
+										.reversed()
+										.thenComparing(Map.Entry::getKey)
+						);
+					} else {
+						// Default alphabetical by key
+						stream = stream.sorted(
+								Comparator.comparing(Map.Entry::getKey)
+						);
+					}
+
+					stream.forEach(entry ->
+							reportSafely(summaryTabIdx, "", entry.getKey(), entry.getValue())
+					);
+				});
+	}
+
 }

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java

@@ -869,8 +869,10 @@ public Collection<RefsetMember> searchMembers(String branchPath, List<String> re
 
 	public Collection<RefsetMember> findRefsetMembers(String branchPath, String refsetId, Boolean isNullEffectiveTime) throws TermServerScriptException {
 		try {
-			Collection<RefsetMember> members = new ArrayList<>(); 
-			String url = getRefsetMembersUrl(branchPath) + "?";
+			Collection<RefsetMember> members = new ArrayList<>();
+			// limit=10000 keeps large refsets (8K+ members) to a single page; the
+			// searchAfter loop below still handles any overflow.
+			String url = getRefsetMembersUrl(branchPath) + "?limit=10000";
 			if (refsetId != null) {
 				url += "&referenceSet=" + refsetId;
 			}

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java

@@ -1901,6 +1901,12 @@ public RefsetMember getOtherRefsetMember(String id) {
 		return null;
 	}
 
+	public List<RefsetMember> getOtherRefsetMembers(String refsetId) {
+		return getOtherRefsetMembers().stream()
+				.filter(m -> refsetId.equals(m.getRefsetId()))
+				.toList();
+	}
+
 	//Set the same axiom details on all stated relationships - if possible
 	public void normalizeStatedRelationships() {
 		AxiomEntry axiomEntry = null;

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/ExecutionOptions.java

@@ -5,6 +5,8 @@ public class ExecutionOptions {
 	public static final ExecutionOptions DEFAULT = new ExecutionOptions();
 
 	boolean doSnapshotImport = true;
+	boolean doIntegrityChecking = true;
+	boolean importAllRefsets = false;
 
 	public boolean isSnapshotImport() {
 		return doSnapshotImport;
@@ -14,4 +16,22 @@ public ExecutionOptions withNoSnapshotImport() {
 		doSnapshotImport = false;
 		return this;
 	}
+
+	public ExecutionOptions withNoIntegrityChecking() {
+		doIntegrityChecking = false;
+		return this;
+	}
+
+	public boolean isIntegrityChecking() {
+		return doIntegrityChecking;
+	}
+
+	public boolean isImportAllRefsets() {
+		return importAllRefsets;
+	}
+
+	public ExecutionOptions withImportAllRefsets() {
+		importAllRefsets = true;
+		return this;
+	}
 }

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/HistoricData.java

@@ -8,6 +8,7 @@
 public class HistoricData implements RF2Constants {
 	private long conceptId;
 	private String fsn;
+	private String usPT;
 	private boolean isActive;
 	private boolean isSD;
 	private String hierarchy;
@@ -45,14 +46,18 @@ public void setConceptId(long conceptId) {
 		this.conceptId = conceptId;
 	}
 
-	public String getFsn() {
-		return fsn;
-	}
+	public String getFsn() { return fsn; }
 
 	public void setFsn(String fsn) {
 		this.fsn = fsn;
 	}
 
+	public String getUsPT() { return usPT; }
+
+	public void setUsPT(String usPT) {
+		this.usPT = usPT;
+	}
+
 	public boolean isActive() {
 		return isActive;
 	}
@@ -293,6 +298,7 @@ public static HistoricData fromLine(String line, boolean minimalSet) {
 		datum.conceptId = Long.parseLong(lineItems[idx]);
 		datum.hashCode = Long.hashCode(datum.conceptId);
 		datum.fsn = lineItems[++idx];
+		datum.usPT = lineItems[++idx];
 		datum.isActive = lineItems[++idx].equals("Y");
 		if (!minimalSet) {
 			datum.isSD = lineItems[++idx].equals("SD");

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/RelationshipGroup.java

@@ -350,14 +350,37 @@ public List<org.snomed.otf.owltoolkit.domain.Relationship> getToolKitRelationshi
 		for (IRelationship r : getIRelationships()) {
 			int groupId = r.getGroupId();
 			long type = Long.parseLong(r.getType().getConceptId());
-			long target = Long.parseLong(r.getTarget().getConceptId());
-			org.snomed.otf.owltoolkit.domain.Relationship toolKitRel = 
-					new org.snomed.otf.owltoolkit.domain.Relationship(groupId, type, target);
-			toolkitRels.add(toolKitRel);
+			if (r.getTarget() == null) {
+				org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue cv = getToolKitConcreteValue(r);
+				org.snomed.otf.owltoolkit.domain.Relationship toolKitRel =
+						new org.snomed.otf.owltoolkit.domain.Relationship(groupId, type, cv);
+				toolkitRels.add(toolKitRel);
+			} else {
+				long target = Long.parseLong(r.getTarget().getConceptId());
+				org.snomed.otf.owltoolkit.domain.Relationship toolKitRel =
+						new org.snomed.otf.owltoolkit.domain.Relationship(groupId, type, target);
+				toolkitRels.add(toolKitRel);
+			}
 		}
 		return toolkitRels;
 	}
 
+	private org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue getToolKitConcreteValue(IRelationship r) {
+		ConcreteValue cv = r.getConcreteValue();
+		if (cv == null) {
+			throw new IllegalArgumentException("Relationship has neither target, nor concrete value");
+		}
+		return new org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue(getToolKitConcreteValueType(cv), cv.getValue());
+	}
+
+	private org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type getToolKitConcreteValueType(ConcreteValue cv) {
+		return switch (cv.getDataType()) {
+			case DECIMAL -> org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type.DECIMAL;
+			case INTEGER -> org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type.INTEGER;
+			case STRING -> org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type.STRING;
+		};
+	}
+
 	public Concept getSourceConcept() {
 		//Can return any relationship's source concept - they will all be the same
 		return ensureRelationship(relationships.iterator().next()).getSource();

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/TermServerReport.java

@@ -206,6 +206,22 @@ private String isActive (Component c) {
 		return c.isActiveSafely() ? "Y" : "N";
 	}
 
+	protected void reportAndIncrementCategoryCount(int tabIdx, String category, boolean isIssueToCount, Concept c, boolean isLegacy, Object... details) throws TermServerScriptException {
+		//Are we filtering this report to only concepts with unpromoted changes?
+		if (unpromotedChangesOnly && !unpromotedChangesHelper.hasUnpromotedChange(c)) {
+			return;
+		}
+
+		if (includeLegacyIssues || !isLegacy) {
+			//The first detail is the issue text
+			incrementSummaryCount(category, details[0].toString());
+			if (report(tabIdx, c, details) && isIssueToCount) {
+				countIssue(c);
+				incrementSummaryCount("Issue Type Summary", (isLegacy?"Legacy Issues Reported": "Fresh Issues Reported"));
+			}
+		}
+	}
+
 	protected void reportAndIncrementSummary(Concept c, boolean isLegacy, Object... details) throws TermServerScriptException {
 		//Are we filtering this report to only concepts with unpromoted changes?
 		if (unpromotedChangesOnly && !unpromotedChangesHelper.hasUnpromotedChange(c)) {

reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java

@@ -300,6 +300,7 @@ public void init (JobRun run) throws TermServerScriptException {
 			populateTemplates(null, "templates/procedures/Endoscopy.json");
 			populateTemplates(null, "templates/procedures/Excision of cyst.json");
 			populateTemplates(null, "templates/procedures/Exteriorization.json");
+			populateTemplates(null, "templates/procedures/FluroAngiograpyAndStent.json");
 			populateTemplates(null, "templates/procedures/Imaging Guided Biopsy.json");
 			populateTemplates(null, "templates/procedures/InsertionOfStent.json");
 			populateTemplates(null, "templates/procedures/Intubation.json");
@@ -309,7 +310,6 @@ public void init (JobRun run) throws TermServerScriptException {
 			populateTemplates(null, "templates/procedures/Radiotherapy.json");
 			populateTemplates(null, "templates/procedures/Stoma.json");
 
-
 			//Do this one last to pick up whatever is left under Disease
 			populateTemplates("<< 64572001 |Disease (disorder)|", "templates/Disease.json");