diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java index 0f6a04dfc7..b031776f0f 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/GraphLoader.java @@ -116,20 +116,15 @@ private static void populateKnownConcepts() { public Collection getAllConcepts() { return concepts.values(); } - public void reset() { - LOGGER.info("Resetting Graph Loader - configuration reset"); - setRecordPreviousState(false); - memoryWipe(); - System.gc(); outputMemoryUsage(); } public void memoryWipe() { - LOGGER.info("Resetting Graph Loader - memory wipe"); + LOGGER.info("Resetting Graph Loader - memory wipe, retained configuration"); concepts = new HashMap<>(); mdrs = null; descriptions = new HashMap<>(); @@ -717,7 +712,7 @@ public void loadConceptFile(InputStream is, Boolean isReleased) throws IOExcepti //We might already have received some details about this concept Concept c = getConcept(lineItems[IDX_ID]); - + //If moduleId is null, then this concept has no prior state if (isRecordPreviousState()) { if (isReleased == null) { diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java index f4a39d6102..11c222f8bd 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/TermServerScript.java @@ -8,6 +8,7 @@ import java.util.stream.Stream; import org.ihtsdo.otf.rest.client.terminologyserver.pojo.*; +import org.ihtsdo.otf.utils.SnomedUtilsBase; import org.ihtsdo.termserver.scripting.dao.ReportDataBroker; import org.apache.commons.lang.time.DurationFormatUtils; @@ -138,6 +139,8 @@ public abstract class TermServerScript extends Script implements ScriptConstants protected ReportDataBroker reportDataBroker; + private Map> summaryCountsByCategory = new HashMap<>(); + public static Gson gson; static { GsonBuilder gsonBuilder = new GsonBuilder(); @@ -611,7 +614,7 @@ protected void preInit() throws TermServerScriptException { //like selfDetermining = true; } - protected void runJob () throws TermServerScriptException { + public void runJob () throws TermServerScriptException { throw new TermServerScriptException("Override this method in concrete class"); } @@ -1844,27 +1847,49 @@ protected boolean inScope(Component c, boolean includeExpectedExtensionModules) if (project.getKey().equals("MAIN")) { return true; } - //Do we have a default module id ie for a managed service project? - if (project.getMetadata() != null && project.getMetadata().getDefaultModuleId() != null) { - //We really need to be sure that expectedExtensionModules has been populated, - //because CH and NO will have content in multiple modules - if (project.getMetadata().getExpectedExtensionModules() == null) { - if (allowMissingExpectedModules) { - return c.getModuleId().equals(project.getMetadata().getDefaultModuleId()); - } else { - throw new IllegalArgumentException("Extension does not have expectedExtensionModules metadata populated. Cannot continue."); - } - } - if (includeExpectedExtensionModules) { - return project.getMetadata().getExpectedExtensionModules().contains(c.getModuleId()); - } else { - return c.getModuleId().equals(project.getMetadata().getDefaultModuleId()); - } + + List inScopeModules = getInScopeModules(includeExpectedExtensionModules); + if (!inScopeModules.isEmpty()) { + return inScopeModules.contains(c.getModuleId()); } else if (moduleFilter != null) { return moduleFilter.contains(c.getModuleId()); } return true; } + + protected List getInScopeModules(boolean includeExpectedExtensionModules) { + if (project.getMetadata() == null) { + return List.of(); + } + + var metadata = project.getMetadata(); + var defaultModuleId = metadata.getDefaultModuleId(); + + if (defaultModuleId == null) { + return List.of(); + } + + var expectedModules = metadata.getExpectedExtensionModules(); + if (expectedModules == null) { + if (!allowMissingExpectedModules) { + throw new IllegalArgumentException( + "Extension does not have expectedExtensionModules metadata populated. Cannot continue." + ); + } + return List.of(defaultModuleId); + } + + return includeExpectedExtensionModules + ? expectedModules + : List.of(defaultModuleId); + } + + protected Set getInScopeNamespaces() { + return getInScopeModules(true).stream() + .map(SnomedUtilsBase::getNamespace) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } protected boolean isMS() { //Do we have a default module id ie for a managed service project? @@ -2361,7 +2386,7 @@ protected void shuffleDown(Task t, Concept c) throws TermServerScriptException { List newGroups = new ArrayList<>(); for (RelationshipGroup group : c.getRelationshipGroups(CharacteristicType.STATED_RELATIONSHIP)) { //Have we missed out the ungrouped group? fill in if so - if (group.isGrouped() && newGroups.size() == 0) { + if (group.isGrouped() && newGroups.isEmpty()) { newGroups.add(new RelationshipGroup(UNGROUPED)); } //Since we're working with the true concept relationships here, this will have @@ -2375,7 +2400,7 @@ protected void shuffleDown(Task t, Concept c) throws TermServerScriptException { for (Relationship moved : new ArrayList<>(group.getRelationships())) { if (StringUtils.isEmpty(moved.getId())) { Set existingInactives = c.getRelationships(moved, ActiveState.INACTIVE); - if (existingInactives.size() > 0) { + if (!existingInactives.isEmpty()) { group.removeRelationship(moved); c.removeRelationship(moved, true); //It's OK to force removal, the axiom will still exist. Relationship reuse = existingInactives.iterator().next(); @@ -2400,4 +2425,60 @@ public boolean isDryRun() { return dryRun; } + public void initialiseSummaryCount(String category, String item) { + summaryCountsByCategory + .computeIfAbsent(category, k -> new HashMap<>()) + .putIfAbsent(item, 0); + } + + public void incrementSummaryCount(String category, String summaryItem) { + incrementSummaryCount(category, summaryItem, 1); + } + + public void incrementSummaryCount(String category, String summaryItem, int increment) { + //Increment the count for this summary item, in the appropriate category + Map summaryCounts = summaryCountsByCategory.computeIfAbsent(category, k -> new HashMap<>()); + summaryCounts.merge(summaryItem, increment, Integer::sum); + } + + public enum SUMMARY_SORT_ORDER { + ALPHABETICAL, + COUNT + } + + protected void reportSummaryCounts(int summaryTabIdx) throws TermServerScriptException { + reportSummaryCounts(summaryTabIdx, SUMMARY_SORT_ORDER.ALPHABETICAL); + } + + protected void reportSummaryCounts(int summaryTabIdx, SUMMARY_SORT_ORDER sortOrder) throws TermServerScriptException { + report(summaryTabIdx, ""); + + // Work through each category (sorted alphabetically) + summaryCountsByCategory.keySet().stream() + .sorted() + .forEach(cat -> { + reportSafely(summaryTabIdx, cat); + + Map summaryCounts = summaryCountsByCategory.get(cat); + Stream> stream = summaryCounts.entrySet().stream(); + if (sortOrder == SUMMARY_SORT_ORDER.COUNT) { + // Sort by count descending, then alphabetically for stability + stream = stream.sorted( + Comparator.comparing(Map.Entry::getValue) + .reversed() + .thenComparing(Map.Entry::getKey) + ); + } else { + // Default alphabetical by key + stream = stream.sorted( + Comparator.comparing(Map.Entry::getKey) + ); + } + + stream.forEach(entry -> + reportSafely(summaryTabIdx, "", entry.getKey(), entry.getValue()) + ); + }); + } + } diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java index 4eff425ab9..4e854c24fc 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/client/TermServerClient.java @@ -869,8 +869,10 @@ public Collection searchMembers(String branchPath, List re public Collection findRefsetMembers(String branchPath, String refsetId, Boolean isNullEffectiveTime) throws TermServerScriptException { try { - Collection members = new ArrayList<>(); - String url = getRefsetMembersUrl(branchPath) + "?"; + Collection members = new ArrayList<>(); + // limit=10000 keeps large refsets (8K+ members) to a single page; the + // searchAfter loop below still handles any overflow. + String url = getRefsetMembersUrl(branchPath) + "?limit=10000"; if (refsetId != null) { url += "&referenceSet=" + refsetId; } diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java index 2a3c6317d5..32f9178b08 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/Concept.java @@ -1901,6 +1901,12 @@ public RefsetMember getOtherRefsetMember(String id) { return null; } + public List getOtherRefsetMembers(String refsetId) { + return getOtherRefsetMembers().stream() + .filter(m -> refsetId.equals(m.getRefsetId())) + .toList(); + } + //Set the same axiom details on all stated relationships - if possible public void normalizeStatedRelationships() { AxiomEntry axiomEntry = null; diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/ExecutionOptions.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/ExecutionOptions.java index 921d8135c9..846dfdb4a7 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/ExecutionOptions.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/ExecutionOptions.java @@ -5,6 +5,8 @@ public class ExecutionOptions { public static final ExecutionOptions DEFAULT = new ExecutionOptions(); boolean doSnapshotImport = true; + boolean doIntegrityChecking = true; + boolean importAllRefsets = false; public boolean isSnapshotImport() { return doSnapshotImport; @@ -14,4 +16,22 @@ public ExecutionOptions withNoSnapshotImport() { doSnapshotImport = false; return this; } + + public ExecutionOptions withNoIntegrityChecking() { + doIntegrityChecking = false; + return this; + } + + public boolean isIntegrityChecking() { + return doIntegrityChecking; + } + + public boolean isImportAllRefsets() { + return importAllRefsets; + } + + public ExecutionOptions withImportAllRefsets() { + importAllRefsets = true; + return this; + } } diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/HistoricData.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/HistoricData.java index 45dc050116..d14e716616 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/HistoricData.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/HistoricData.java @@ -8,6 +8,7 @@ public class HistoricData implements RF2Constants { private long conceptId; private String fsn; + private String usPT; private boolean isActive; private boolean isSD; private String hierarchy; @@ -45,14 +46,18 @@ public void setConceptId(long conceptId) { this.conceptId = conceptId; } - public String getFsn() { - return fsn; - } + public String getFsn() { return fsn; } public void setFsn(String fsn) { this.fsn = fsn; } + public String getUsPT() { return usPT; } + + public void setUsPT(String usPT) { + this.usPT = usPT; + } + public boolean isActive() { return isActive; } @@ -293,6 +298,7 @@ public static HistoricData fromLine(String line, boolean minimalSet) { datum.conceptId = Long.parseLong(lineItems[idx]); datum.hashCode = Long.hashCode(datum.conceptId); datum.fsn = lineItems[++idx]; + datum.usPT = lineItems[++idx]; datum.isActive = lineItems[++idx].equals("Y"); if (!minimalSet) { datum.isSD = lineItems[++idx].equals("SD"); diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/RelationshipGroup.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/RelationshipGroup.java index 6f979200f7..53ebeb792c 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/RelationshipGroup.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/domain/RelationshipGroup.java @@ -350,14 +350,37 @@ public List getToolKitRelationshi for (IRelationship r : getIRelationships()) { int groupId = r.getGroupId(); long type = Long.parseLong(r.getType().getConceptId()); - long target = Long.parseLong(r.getTarget().getConceptId()); - org.snomed.otf.owltoolkit.domain.Relationship toolKitRel = - new org.snomed.otf.owltoolkit.domain.Relationship(groupId, type, target); - toolkitRels.add(toolKitRel); + if (r.getTarget() == null) { + org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue cv = getToolKitConcreteValue(r); + org.snomed.otf.owltoolkit.domain.Relationship toolKitRel = + new org.snomed.otf.owltoolkit.domain.Relationship(groupId, type, cv); + toolkitRels.add(toolKitRel); + } else { + long target = Long.parseLong(r.getTarget().getConceptId()); + org.snomed.otf.owltoolkit.domain.Relationship toolKitRel = + new org.snomed.otf.owltoolkit.domain.Relationship(groupId, type, target); + toolkitRels.add(toolKitRel); + } } return toolkitRels; } + private org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue getToolKitConcreteValue(IRelationship r) { + ConcreteValue cv = r.getConcreteValue(); + if (cv == null) { + throw new IllegalArgumentException("Relationship has neither target, nor concrete value"); + } + return new org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue(getToolKitConcreteValueType(cv), cv.getValue()); + } + + private org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type getToolKitConcreteValueType(ConcreteValue cv) { + return switch (cv.getDataType()) { + case DECIMAL -> org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type.DECIMAL; + case INTEGER -> org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type.INTEGER; + case STRING -> org.snomed.otf.owltoolkit.domain.Relationship.ConcreteValue.Type.STRING; + }; + } + public Concept getSourceConcept() { //Can return any relationship's source concept - they will all be the same return ensureRelationship(relationships.iterator().next()).getSource(); diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/TermServerReport.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/TermServerReport.java index 0ccd5f3c56..b94922c8e1 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/TermServerReport.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/TermServerReport.java @@ -206,6 +206,22 @@ private String isActive (Component c) { return c.isActiveSafely() ? "Y" : "N"; } + protected void reportAndIncrementCategoryCount(int tabIdx, String category, boolean isIssueToCount, Concept c, boolean isLegacy, Object... details) throws TermServerScriptException { + //Are we filtering this report to only concepts with unpromoted changes? + if (unpromotedChangesOnly && !unpromotedChangesHelper.hasUnpromotedChange(c)) { + return; + } + + if (includeLegacyIssues || !isLegacy) { + //The first detail is the issue text + incrementSummaryCount(category, details[0].toString()); + if (report(tabIdx, c, details) && isIssueToCount) { + countIssue(c); + incrementSummaryCount("Issue Type Summary", (isLegacy?"Legacy Issues Reported": "Fresh Issues Reported")); + } + } + } + protected void reportAndIncrementSummary(Concept c, boolean isLegacy, Object... details) throws TermServerScriptException { //Are we filtering this report to only concepts with unpromoted changes? if (unpromotedChangesOnly && !unpromotedChangesHelper.hasUnpromotedChange(c)) { diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java index 0975d0e9d8..1a15145349 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/qi/AllKnownTemplates.java @@ -300,6 +300,7 @@ public void init (JobRun run) throws TermServerScriptException { populateTemplates(null, "templates/procedures/Endoscopy.json"); populateTemplates(null, "templates/procedures/Excision of cyst.json"); populateTemplates(null, "templates/procedures/Exteriorization.json"); + populateTemplates(null, "templates/procedures/FluroAngiograpyAndStent.json"); populateTemplates(null, "templates/procedures/Imaging Guided Biopsy.json"); populateTemplates(null, "templates/procedures/InsertionOfStent.json"); populateTemplates(null, "templates/procedures/Intubation.json"); @@ -309,7 +310,6 @@ public void init (JobRun run) throws TermServerScriptException { populateTemplates(null, "templates/procedures/Radiotherapy.json"); populateTemplates(null, "templates/procedures/Stoma.json"); - //Do this one last to pick up whatever is left under Disease populateTemplates("<< 64572001 |Disease (disorder)|", "templates/Disease.json"); diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java index b067c64cf3..400e2120f0 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricStatsGenerator.java @@ -7,6 +7,7 @@ import java.util.*; import java.util.stream.Collectors; +import org.ihtsdo.otf.RF2Constants; import org.ihtsdo.otf.exception.TermServerScriptException; import org.ihtsdo.otf.rest.client.terminologyserver.pojo.Component; import org.ihtsdo.otf.rest.client.terminologyserver.pojo.ComponentAnnotationEntry; @@ -145,6 +146,8 @@ private void generateHistoricData(FileWriter fw, TransitiveClosure tc) throws Te for (Concept c : gl.getAllConcepts()) { String active = c.isActiveSafely() ? "Y" : "N"; + Description usPTDesc = c.getPreferredSynonym(RF2Constants.US_ENG_LANG_REFSET); + String usPT = usPTDesc != null ? usPTDesc.getTerm() : null; String defStatus = SnomedUtils.translateDefnStatus(c.getDefinitionStatus()); String hierarchy = getHierarchy(tc, c, new LinkedList<>()); String intermediatePrimitiveIndicator = intermediatePrimitives.contains(c) ? "Y" : "N"; @@ -161,8 +164,8 @@ private void generateHistoricData(FileWriter fw, TransitiveClosure tc) throws Te String[] annotationIds = getAnnotationIds(c); String hasAttributes = SnomedUtils.countAttributes(c, CharacteristicType.INFERRED_RELATIONSHIP) > 0 ? "Y" : "N"; String histAssocTargets = getHistAssocTargets(c); - ouput(fw, c.getConceptId(), c.getFsn(), active, defStatus, hierarchy, intermediatePrimitiveIndicator, - sdDescendant, sdAncestor, + ouput(fw, c.getConceptId(), c.getFsn(), usPT, active, defStatus, hierarchy, + intermediatePrimitiveIndicator, sdDescendant, sdAncestor, relIds[ACTIVE], relIds[INACTIVE], descIds[ACTIVE], descIds[INACTIVE], axiomIds[ACTIVE], axiomIds[INACTIVE], langRefSetIds[ACTIVE], langRefSetIds[INACTIVE], inactivationIds[ACTIVE], inactivationIds[INACTIVE], histAssocIds[ACTIVE], histAssocIds[INACTIVE], diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/snapshot/ArchiveManager.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/snapshot/ArchiveManager.java index 9d91d745f3..21616ab342 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/snapshot/ArchiveManager.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/snapshot/ArchiveManager.java @@ -826,6 +826,7 @@ public void reset() { LOGGER.info("Temporary stack trace so we can see _why_ we're being reset: {}", stackTrace); gl.reset(); + gl.setRecordPreviousState(false); currentlyHeldInMemory = null; config.reset(); } diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/DrugTermGenerator.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/DrugTermGenerator.java index 0579c190d3..737b0646a8 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/DrugTermGenerator.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/DrugTermGenerator.java @@ -126,7 +126,7 @@ private int ensureDrugTermConforms(Task t, Concept c, Description d, Characteris if (!replacementTerm.equals(d.getTerm())) { changesMade += replaceTerm(t, c, d, replacement, true); } else { - //Are we seeing a difference in the case signficance? + //Are we seeing a difference in the case-significance? if (!d.getCaseSignificance().equals(replacement.getCaseSignificance())) { String before = SnomedUtils.translateCaseSignificanceFromEnum(d.getCaseSignificance()); String after = SnomedUtils.translateCaseSignificanceFromEnum(replacement.getCaseSignificance()); @@ -158,12 +158,14 @@ private int ensureDrugTermConforms(Task t, Concept c, Description d, Characteris private int checkCaseSensitivitySetting(Task t, Concept c, Description d, boolean isFSN, CharacteristicType charType, String langRefset) throws TermServerScriptException { int changesMade = 0; - //Firstly, are there any absolute rules that would force the case signficance like a capital after the first letter + //Firstly, are there any absolute rules that would force the case-significance like a capital after the first letter //or starting with a lower case? if (StringUtils.initialLetterLowerCase(d.getTerm())) { return modifyIfRequired(d, CaseSignificance.ENTIRE_TERM_CASE_SENSITIVE); - //For the other settings, we need to check further rules below as lower case letters might not look case sensitive. - } else if (StringUtils.isCaseSensitive(d.getTerm()) && !d.getCaseSignificance().equals(CaseSignificance.ENTIRE_TERM_CASE_SENSITIVE)) { + //For the other settings, we need to check further rules below as lower case letters might not look case-sensitive. + } else if (DrugUtils.containsKnownCaseSensitiveDrugUnit(d) || ( + StringUtils.isCaseSensitive(d.getTerm()) + && !d.getCaseSignificance().equals(CaseSignificance.ENTIRE_TERM_CASE_SENSITIVE))) { d.setCaseSignificance(CaseSignificance.INITIAL_CHARACTER_CASE_INSENSITIVE); } else if (!StringUtils.isCaseSensitive(d.getTerm()) && !d.getCaseSignificance().equals(CaseSignificance.INITIAL_CHARACTER_CASE_INSENSITIVE)) { d.setCaseSignificance(CaseSignificance.CASE_INSENSITIVE); @@ -214,8 +216,12 @@ private int checkCaseSensitivitySetting(Task t, Concept c, Description d, boolea } isFirstIngred = false; } - //Watch that this doesn't allow for any case sensitivity in the dose form, units, or presentation. Currently there is none... - if (!hasCaseSensitiveIngredient && !d.getCaseSignificance().equals(CaseSignificance.CASE_INSENSITIVE) && !StringUtils.isCaseSensitive(d.getTerm())) { + + //Watch that this doesn't allow for any case sensitivity in the dose form or presentation. Currently, there is none... + if (!DrugUtils.containsKnownCaseSensitiveDrugUnit(d) + && !hasCaseSensitiveIngredient + && !d.getCaseSignificance().equals(CaseSignificance.CASE_INSENSITIVE) + && !StringUtils.isCaseSensitive(d.getTerm())) { report(t, c, Severity.MEDIUM, ReportActionType.VALIDATION_CHECK, "Set term to ci due to lack of case sensitivity in any ingredient"); d.setCaseSignificance(CaseSignificance.CASE_INSENSITIVE); changesMade++; diff --git a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java index ac9251cebb..b1f32ed8eb 100644 --- a/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java +++ b/reporting-core/src/main/java/org/ihtsdo/termserver/scripting/util/SnomedUtils.java @@ -34,6 +34,8 @@ public class SnomedUtils extends SnomedUtilsBase implements ScriptConstants { private static final Logger LOGGER = LoggerFactory.getLogger(SnomedUtils.class); + private static final Set KNOWN_WRONG_PARTITION_IDS = Set.of("10751000009126"); + private static final SimpleDateFormat EFFECTIVE_DATE_FORMAT = new SimpleDateFormat("yyyyMMdd"); private static VerhoeffCheckDigit verhoeffCheck = new VerhoeffCheckDigit(); @@ -77,7 +79,11 @@ public static boolean isValid(String sctId, PartitionIdentifier partitionIdentif boolean errorIfInvalid) throws TermServerScriptException { String errMsg = isValid(sctId,partitionIdentifier); if (errorIfInvalid && errMsg != null) { - throw new TermServerScriptException(errMsg); + if (KNOWN_WRONG_PARTITION_IDS.contains(sctId)) { + LOGGER.warn("Allowing known wrong partition for {}", errMsg); + } else { + throw new TermServerScriptException(errMsg); + } } return errMsg == null; } diff --git a/reporting-core/src/main/resources/templates/procedures/FluroAngiographyAndStent.json b/reporting-core/src/main/resources/templates/procedures/FluroAngiographyAndStent.json new file mode 100644 index 0000000000..505103ad66 --- /dev/null +++ b/reporting-core/src/main/resources/templates/procedures/FluroAngiographyAndStent.json @@ -0,0 +1,7 @@ +{ + "name": "Fluoroscopic angiography and insertion of stent (procedure)", + "documentation": "https://conf.spaces.snomed.org/wiki/spaces/SCTEMPLATES/pages/134000396/Fluoroscopic+angiography+and+insertion+of+stent+procedure", + "version": 1, + "domain": "<< 1296925008 |Insertion of stent (procedure)|: 260686004 |Method| = << 312275004 |Fluoroscopic imaging - action (qualifier value)|, [0..2] 363703001 |Has intent (attribute)| = 429892002 |Guidance intent (qualifier value)|, 405814001 |Procedure site - Indirect| = <<59820001 |Blood vessel structure (body structure)|", + "logicalTemplate": "71388002 |Procedure (procedure)| : [[~1..* @AngioRoleGroup]]{ [[~1..1]] 260686004 |Method (attribute)| = [[ +id ( 312275004 |Fluoroscopic imaging - action (qualifier value)| ) ]] , [[~1..1]] 405813007 |Procedure site - Direct (attribute)| = [[ +id ( << 59820001 |Blood vessel structure (body structure)| ) ]] , [[~1..1]] 424361007 |Using substance (attribute)| = [[ +id ( << 385420005 |Contrast media (substance)| ) ]] , [[~0..1]] 363700003 |Direct morphology (attribute)| = [[ +id ( < 49755003 |Morphologically abnormal structure (morphologic abnormality)| ) ]] , [[~1..1]] 363699004 |Direct device (attribute)| = [[ +id ( << 65818007 |Stent (physical object)| ) ]] , [[~1..1]] 363703001 |Has intent (attribute)| = [[ +id ( 429892002 |Guidance intent (qualifier value)| ) ]] }, [[~1..* @StentRoleGroup]]{ [[~1..1]] 260686004 |Method (attribute)| = [[ +id ( << 257867005 |Insertion - action (qualifier value)| ) ]] , [[~1..1]] 405814001 |Procedure site - Indirect (attribute)| = [[ +id ( << 59820001 |Blood vessel structure (body structure)| ) ]] , [[~1..1]] 363699004 |Direct device (attribute)| = [[ +id ( << 65818007 |Stent (physical object)| ) ]], [[~0..1]] 260507000 |Access (attribute)| = [[ +id ( << 309795001 |Surgical access values (qualifier value)| ) ]] , [[~0..1]] 363709002 |Indirect morphology (attribute)| = [[ +id ( < 49755003 |Morphologically abnormal structure (morphologic abnormality)| ) ]] , [[~0..1]] 116688005 |Procedure approach (attribute)| = [[ +id ( << 103379005 |Procedural approach (qualifier value)| ) ]]}, [[~0..* @DilationRoleGroup]]{ [[~1..1]] 260686004 |Method (attribute)| = [[ +id ( 410817004 |Dilation repair - action (qualifier value)| ) ]] , [[~1..1]] 405813007 |Procedure site - Direct (attribute)| = [[ +id ( << 59820001 |Blood vessel structure (body structure)| ) ]] , [[~0..1]] 260507000 |Access (attribute)| = [[ +id ( << 309795001 |Surgical access values (qualifier value)| ) ]] , [[~1..1]] 424226004 |Using device (attribute)| = [[ +id ( <<310362005 |Angioplasty catheter (physical object)| ) ]] , [[~0..1]] 116688005 |Procedure approach (attribute)| = [[ +id ( << 103379005 |Procedural approach (qualifier value)| ) ]]}" +} \ No newline at end of file diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/drugs/ValidateDrugModelingLegacyReport.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/drugs/ValidateDrugModelingLegacyReport.java index 7e8b91116c..7e8f42ab01 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/drugs/ValidateDrugModelingLegacyReport.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/drugs/ValidateDrugModelingLegacyReport.java @@ -24,6 +24,9 @@ public class ValidateDrugModelingLegacyReport extends TermServerReport implement private static final Logger LOGGER = LoggerFactory.getLogger(ValidateDrugModelingLegacyReport.class); + private static final String PROCESSING_COUNTS = "Processing Counts"; + private static final String ISSUE_COUNTS = "Issue Counts"; + private List allDrugs; private static final String RECENT_CHANGES_ONLY = "Recent Changes Only"; @@ -51,14 +54,14 @@ public class ValidateDrugModelingLegacyReport extends TermServerReport implement Set presAttributes = new HashSet<>(); Set concAttributes = new HashSet<>(); - TermGenerator termGenerator = new DrugTermGenerator(this); + TermGenerator termGenerator; private static final String INJECTION = "injection"; private static final String INFUSION = "infusion"; public static void main(String[] args) throws TermServerScriptException { Map params = new HashMap<>(); - params.put(RECENT_CHANGES_ONLY, "false"); + params.put(RECENT_CHANGES_ONLY, "true"); TermServerScript.run(ValidateDrugModelingLegacyReport.class, args, params); } @@ -67,6 +70,7 @@ public void init (JobRun run) throws TermServerScriptException { ReportSheetManager.setTargetFolderId("1wtB15Soo-qdvb0GHZke9o_SjFSL_fxL3"); //DRUGS/Validation additionalReportColumns = "FSN, SemTag, Issue, Data, Detail"; //DRUGS-267 super.init(run); + getArchiveManager().setPopulateReleaseFlag(true); } @Override @@ -80,6 +84,8 @@ public void postInit() throws TermServerScriptException { doseFormHelper.initialise(gl); populateGrouperSubstances(); populateBaseMDFMap(); + + termGenerator = new DrugTermGenerator(this); super.postInit(tabNames, columnHeadings); @@ -128,7 +134,7 @@ public Job getJob() { public void runJob() throws TermServerScriptException { validateDrugsModeling(); valiadteTherapeuticRole(); - populateSummaryTabAndTotal(SECONDARY_REPORT); + reportSummaryCounts(SECONDARY_REPORT); LOGGER.info("Summary tab complete, all done."); } @@ -136,9 +142,10 @@ private void validateDrugsModeling() throws TermServerScriptException { ConceptType[] allDrugTypes = new ConceptType[] { ConceptType.MEDICINAL_PRODUCT, ConceptType.MEDICINAL_PRODUCT_ONLY, ConceptType.MEDICINAL_PRODUCT_FORM, ConceptType.MEDICINAL_PRODUCT_FORM_ONLY, ConceptType.CLINICAL_DRUG }; ConceptType[] cds = new ConceptType[] { ConceptType.CLINICAL_DRUG }; //DRUGS-267 double conceptsConsidered = 0; - //for (Concept c : Collections.singleton(gl.getConcept("776935006"))) { + for (Concept c : allDrugs) { - if (isRecentlyTouchedConceptsOnly && !recentlyTouchedConcepts.contains(c)) { + if (isRecentlyTouchedConceptsOnly + && !recentlyTouchedConcepts.contains(c)) { continue; } @@ -161,9 +168,6 @@ private void validateDrugsModeling() throws TermServerScriptException { if (isMP(c) || isMPF(c)) { //DRUGS-585 validateNoModifiedSubstances(c); - - //RP-199 - checkForRedundantConcept(c); } //DRUGS-784 @@ -249,15 +253,6 @@ private void validateDrugsModeling() throws TermServerScriptException { LOGGER.info("Drugs validation complete"); } - private void checkForRedundantConcept(Concept c) throws TermServerScriptException { - //MP / MP with no inferred descendants are not required - String issueStr = "MP/MPF concept is redundant - no inferred descendants"; - initialiseSummary(issueStr); - if (c.getDescendants(NOT_SET).isEmpty()) { - report(c, issueStr); - } - } - private void checkForPrimitives(Concept c) throws TermServerScriptException { String issueStr = "Primitive concept"; initialiseSummary(issueStr); @@ -699,7 +694,7 @@ private void validateIngredientsAgainstBoSS(Concept concept) throws TermServerSc //Check BOSS attributes against active ingredients - must be in the same relationship group Set ingredientRels = concept.getRelationships(CharacteristicType.STATED_RELATIONSHIP, HAS_PRECISE_INGRED, ActiveState.ACTIVE); for (Relationship bRel : bossAttributes) { - incrementSummaryInformation("BoSS attributes checked"); + incrementSummaryCount(PROCESSING_COUNTS, "BoSS attributes checked"); boolean matchFound = false; Concept boSS = bRel.getTarget(); for (Relationship iRel : ingredientRels) { @@ -712,12 +707,11 @@ private void validateIngredientsAgainstBoSS(Concept concept) throws TermServerSc if (isSelf || isSubType || isModificationOf) { matchFound = true; if (isSubType) { - incrementSummaryInformation("Active ingredient is a subtype of BoSS"); report(concept, issueStr, ingred, boSS); } else if (isModificationOf) { - incrementSummaryInformation("Valid Ingredients as Modification of BoSS"); + incrementSummaryCount(PROCESSING_COUNTS, "Valid Ingredients as Modification of BoSS"); } else if (isSelf) { - incrementSummaryInformation("BoSS matches ingredient"); + incrementSummaryCount(PROCESSING_COUNTS, "BoSS matches ingredient"); } } } @@ -816,9 +810,9 @@ private Concept getMDF(Concept concept, boolean allowNull) { private void validateTerming(Concept c, ConceptType[] drugTypes) throws TermServerScriptException { //Only check FSN for certain drug types (to be expanded later) if (!SnomedUtils.isConceptType(c, drugTypes)) { - incrementSummaryInformation("Concepts ignored - wrong type"); + incrementSummaryCount(PROCESSING_COUNTS, "Concepts ignored - wrong type"); } - incrementSummaryInformation("Concepts validated to ensure ingredients correct in FSN"); + incrementSummaryCount(PROCESSING_COUNTS, "Concepts validated to ensure ingredients correct in FSN"); Description currentFSN = c.getFSNDescription(); termGenerator.setQuiet(true); @@ -1224,7 +1218,7 @@ private void checkCdUnitConsistency(Concept c) throws TermServerScriptException } else if (!unitOfPres.equals(presDenomUnits.iterator().next().getTarget())) { report(c, issueStr3, unitOfPres, g); } - incrementSummaryInformation("CD groups checked for presentation unit consistency"); + incrementSummaryCount(PROCESSING_COUNTS, "CD groups checked for presentation unit consistency"); } } } @@ -1328,7 +1322,7 @@ private int getTagLevel(Concept c) { @Override public boolean report(Concept c, Object...details) throws TermServerScriptException { //First detail is the issue - issueSummaryMap.merge(details[0].toString(), 1, Integer::sum); + incrementSummaryCount(ISSUE_COUNTS, details[0].toString()); countIssue(c); return super.report(PRIMARY_REPORT, c, details); } diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managed_service/ExtensionImpactReport.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managed_service/ExtensionImpactReport.java index 8658300790..f02d15010a 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managed_service/ExtensionImpactReport.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/managed_service/ExtensionImpactReport.java @@ -3,6 +3,7 @@ import java.util.*; import java.util.stream.Collectors; +import org.ihtsdo.otf.RF2Constants; import org.ihtsdo.otf.exception.TermServerScriptException; import org.ihtsdo.otf.rest.client.terminologyserver.pojo.Component; import org.ihtsdo.otf.rest.client.terminologyserver.pojo.Project; @@ -22,6 +23,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * The important point to make about this report is that the 'history' (calculated first) is actually the state + * of the incoming upgrade package. + */ public class ExtensionImpactReport extends HistoricDataUser implements ReportClass { private static final Logger LOGGER = LoggerFactory.getLogger(ExtensionImpactReport.class); @@ -30,6 +35,9 @@ public class ExtensionImpactReport extends HistoricDataUser implements ReportCla private static final String ECL_FILTER = "ECL Filter (optional)"; private static final String COMMON_HEADINGS = "SCTID, FSN, SemTag,"; + private static final String TERM_TYPE_FSN = "FSN"; + private static final String TERM_TYPE_PT = "PT"; + private static final boolean RUN_INTEGRITY_CHECKS = true; //Make false locally if required private String incomingDataKey; @@ -38,7 +46,8 @@ public class ExtensionImpactReport extends HistoricDataUser implements ReportCla private Map> usedInStatedModellingMap; private Map> usedAsStatedParentMap; private Map historicalAssociationStrMap; - + private final Map fsnToSctIdMap = new HashMap<>(); + private String[][] columnNames; //Used for both column names, and to track totals private String proposedUpgrade; private String ecl; @@ -47,8 +56,7 @@ public class ExtensionImpactReport extends HistoricDataUser implements ReportCla public static void main(String[] args) throws TermServerScriptException { Map params = new HashMap<>(); - params.put(INTERNATIONAL_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20250901T120000Z.zip"); - params.put(ECL_FILTER, "^ 1303957004 |Nutrition Care Process Terminology reference set (foundation metadata concept)| or ^ 1157358007 |International Classification for Nursing Practice reference set (foundation metadata concept)| or ^ 816080008 |International Patient Summary (foundation metadata concept)|"); + params.put(INTERNATIONAL_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20260301T120000Z.zip"); TermServerScript.run(ExtensionImpactReport.class, args, params); } @@ -155,22 +163,28 @@ public void postInit() throws TermServerScriptException { ecl = jobRun.getParamValue(ECL_FILTER); columnNames = new String[][] { {"Has Inactivated Stated Parent", "Inactivated Concept Used As Stated Parent", "Has Inactivated Stated Attribute", "Inactivated Concept Used In Stated Modelling", "Has Inactivated Inferred Parent", "Inactivated Concept Used As Inferred Parent", "Inactivated with Extension Axiom"}, - {"New Concept Requires Translation", "Updated FSN Requires Translation", "Updated FSN No Current Translation", "Translated Concept Inactivated - Replacement Requires Translation", "Translated Concept Inactivated - No Replacement Specified"}}; + {"New Concept Requires Translation", "Updated FSN May Require Translation", "Updated FSN Concept Not Translated", "Updated PT Requires Translation", "Updated PT Concept Not Translated", "Translated Concept Inactivated - Replacement Not Translated", "Translated Concept Inactivated - No Replacement Specified"}}; - String[] columnHeadings = new String[] {"Summary Item, Count", + String[] columnHeadings = new String[] { + "Summary Item, Count", COMMON_HEADINGS + formColumnNames(columnNames[0], true), COMMON_HEADINGS + "Impact,Affected Concept,Historical Associations", COMMON_HEADINGS + formColumnNames(columnNames[1], false), - COMMON_HEADINGS + "Impact Information,Existing FSN,Translated FSN,Translated PT", - COMMON_HEADINGS + "Axioms Affected"}; + COMMON_HEADINGS + "Impact Information,Existing Term,Proposed Term,Translation", + COMMON_HEADINGS + "Axioms Affected", + COMMON_HEADINGS + "Incoming International Concept,Duplicate FSN" + }; - String[] tabNames = new String[]{"Summary Counts", //PRIMARY + String[] tabNames = new String[]{ + "Summary Counts", //PRIMARY "Inactivations", //SECONDARY "Inactivation Detail", //TERTIARY "Translations", //QUAD "Translation Detail", //QUINARY - "Non-core Axioms Detail" //SENARY_REPORT + "Non-core Axioms Detail", //SENARY_REPORT + "FSN Duplicates" //SEPTENARY_REPORT }; + super.postInit(GFOLDER_RELEASE_STATS, tabNames, columnHeadings, false); derivativeHelper = new DerivativeHelper(this); @@ -219,7 +233,6 @@ public void runJob() throws TermServerScriptException { //Work through the top level hierarchies List topLevelHierarchies = SnomedUtils.sort(ROOT_CONCEPT.getDescendants(IMMEDIATE_CHILD)); for (Concept topLevelConcept : topLevelHierarchies) { - LOGGER.info("Processing - {}", topLevelConcept); Set thisHierarchy = getHierarchy(topLevelConcept); reportInactivations(topLevelConcept, thisHierarchy, columnNames[0]); reportTranslations(topLevelConcept, thisHierarchy, columnNames[1]); @@ -228,10 +241,12 @@ public void runJob() throws TermServerScriptException { //We can now populate all the of the total columns writeTotalRow(SECONDARY_REPORT, columnNames[0], true); writeTotalRow(QUATERNARY_REPORT, columnNames[1], false); + + reportFsnDuplicates(); } private void reportInactivations(Concept topLevelConcept, Set thisHierarchy, String[] summaryNames) throws TermServerScriptException { - LOGGER.info("Reporting Inactivations"); + LOGGER.info("Reporting inactivations for {}", topLevelConcept); int[] inactivationCounts = new int[7]; String[] examples = new String[4]; @@ -378,9 +393,8 @@ private void countInactivatedConceptsWithNonCoreAxioms(Concept currentConcept, S } private void reportTranslations(Concept topLevelConcept, Set thisHierarchy, String[] summaryNames) throws TermServerScriptException { - LOGGER.info("Reporting Translations Required"); - int[] translationCounts = new int[5]; - + LOGGER.info("Reporting translations required for {}", topLevelConcept); + int[] translationCounts = new int[7]; Set conceptReplacementSeen = new HashSet<>(); for (String sctId : thisHierarchy) { @@ -394,7 +408,9 @@ private void reportTranslations(Concept topLevelConcept, Set thisHierarc translationCounts[1], translationCounts[2], translationCounts[3], - translationCounts[4]); + translationCounts[4], + translationCounts[5], + translationCounts[6]); } private int[] countTranslations(String sctId, String[] summaryNames, Set conceptReplacementSeen) throws TermServerScriptException { @@ -416,6 +432,8 @@ private int[] countTranslations(String sctId, String[] summaryNames, Set translationStats.newConceptCount, translationStats.changedFSNCount, translationStats.changedFSNCountNoCurrent, + translationStats.changedPTCount, + translationStats.changedPTCountNoCurrent, translationStats.translatedInactivatedCount, translationStats.translatedInactivatedWithoutReplacement }; @@ -434,7 +452,10 @@ private void countNewConcepts(HistoricData datum, String[] summaryNames, Transla } private void compareCurrentConceptWithPreviousState(Concept currentConcept, HistoricData datum, String[] summaryNames, TranslationStats translationStats, Set conceptReplacementSeen) throws TermServerScriptException { - checkForChangedFSN(currentConcept, datum, translationStats, summaryNames); + if (isConceptOfInterest(currentConcept.getId())) { + checkForChangedTerm(TERM_TYPE_FSN, currentConcept, datum, translationStats, summaryNames); + checkForChangedTerm(TERM_TYPE_PT, currentConcept, datum, translationStats, summaryNames); + } //Report translated concepts that have been inactivated where the replacement has not been translated if (!datum.isActive() && @@ -445,7 +466,7 @@ private void compareCurrentConceptWithPreviousState(Concept currentConcept, Hist if (datum.getHistAssocTargets() == null) { if (isConceptOfInterest(currentConcept.getId())) { translationStats.translatedInactivatedWithoutReplacement++; - incrementSummaryInformation(summaryNames[4]); + incrementSummaryInformation(summaryNames[6]); String fsn = datum.getFsn(); String semTag = SnomedUtilsBase.deconstructFSN(fsn)[1]; report(QUINARY_REPORT, datum.getConceptId(), fsn, semTag, "Translated concept has been inactivated but no replacement specified", currentConcept.getFsn(), getTranslatedFsn(currentConcept), getTranslatedPreferredSynonym(currentConcept)); @@ -466,7 +487,7 @@ private void checkHistoricalAssociationsForReplacements(HistoricData datum, Tran //If we don't have this concept then it definitely won't have a translation if (targetConcept == null || !hasTranslation(targetConcept)) { translationStats.translatedInactivatedCount++; - incrementSummaryInformation(summaryNames[3]); + incrementSummaryInformation(summaryNames[5]); //We might not know about this concept in this project, but it should be known to the incoming data String parentConcept = datum.getConceptId() + " |" + datum.getFsn() + "|"; HistoricData incomingDatum = incomingData.get(histAssocTarget); @@ -479,19 +500,115 @@ private void checkHistoricalAssociationsForReplacements(HistoricData datum, Tran } } - private void checkForChangedFSN(Concept currentConcept, HistoricData datum, TranslationStats translationStats, String[] summaryNames) throws TermServerScriptException { - //Has the FSN changed from what's currently here? - if (!currentConcept.getFsn().equals(datum.getFsn()) && isConceptOfInterest(currentConcept.getId())) { - String fsn = datum.getFsn(); - String semTag = SnomedUtilsBase.deconstructFSN(fsn)[1]; - if (hasTranslation(currentConcept)) { - report(QUINARY_REPORT, datum.getConceptId(), fsn, semTag, "Existing translated FSN has been replaced", currentConcept.getFsn(), getTranslatedFsn(currentConcept), getTranslatedPreferredSynonym(currentConcept)); + private void checkForChangedTerm(String termType, + Concept currentConcept, + HistoricData incomingState, + TranslationStats translationStats, + String[] summaryNames) throws TermServerScriptException { + + boolean isFSN = TERM_TYPE_FSN.equals(termType); + + Description usPTDesc = currentConcept.getPreferredSynonym(RF2Constants.US_ENG_LANG_REFSET); + String usPT = usPTDesc != null ? usPTDesc.getTerm() : null; + + String currentTerm = isFSN ? currentConcept.getFsn() : usPT; + String incomingTerm = isFSN ? incomingState.getFsn() : incomingState.getUsPT(); + + if (incomingTerm.equals(currentTerm)) { + return; + } + + handleChangedTerm( + isFSN, + termType, + currentConcept, + incomingState, + translationStats, + summaryNames, + currentTerm, + incomingTerm + ); + } + + private void handleChangedTerm(boolean isFSN, + String termType, + Concept currentConcept, + HistoricData incomingState, + TranslationStats translationStats, + String[] summaryNames, + String currentTerm, + String incomingTerm) throws TermServerScriptException { + + boolean translated = hasTranslation(currentConcept); + + String translatedTerm = ""; + if (translated) { + translatedTerm = isFSN + ? getTranslatedFsn(currentConcept) + : getTranslatedPreferredSynonym(currentConcept); + } + + String messagePrefix = translated + ? "Translated concept having " + : "Untranslated concept having "; + String fsn = incomingState.getFsn(); + String semTag = SnomedUtilsBase.deconstructFSN(fsn)[1]; + report(QUINARY_REPORT, + incomingState.getConceptId(), + fsn, + semTag, + messagePrefix + termType + " replaced", + currentTerm, + incomingTerm, + translatedTerm); + + if (isFSN) { + if (translated) { translationStats.changedFSNCount++; - incrementSummaryInformation(summaryNames[1]); } else { - report(QUINARY_REPORT, datum.getConceptId(), fsn, semTag, "Existing untranslated FSN has been replaced", currentConcept.getFsn(), "", ""); translationStats.changedFSNCountNoCurrent++; - incrementSummaryInformation(summaryNames[2]); + } + } else { + if (translated) { + translationStats.changedPTCount++; + } else { + translationStats.changedPTCountNoCurrent++; + } + } + + int outputColumnOffset = isFSN ? 0 : 2; + int summaryIndex = outputColumnOffset + (translated ? 1 : 2); + incrementSummaryInformation(summaryNames[summaryIndex]); + } + + private void lookForDuplicateFsn(Concept c) throws TermServerScriptException { + Description fsnDescription = c.getFSNDescription(); + if (fsnDescription != null) { + String fsn = fsnDescription.getTerm(); + String incomingConceptId = fsnToSctIdMap.get(fsn.toLowerCase()); + if (incomingConceptId != null && !incomingConceptId.equals(c.getConceptId())) { + // If a lower case duplicate is found, check the FSN's case significance + CaseSignificance cs = fsnDescription.getCaseSignificance(); + // Report as a duplicate if either case-insensitive or an exact match + if (CaseSignificance.CASE_INSENSITIVE.equals(cs) || (fsn.equals(incomingData.get(incomingConceptId).getFsn()))) { + report(SEPTENARY_REPORT, c, incomingConceptId, fsn); + incrementSummaryInformation("FSN Duplicates"); + } + } + } + } + + private void reportFsnDuplicates() throws TermServerScriptException { + LOGGER.info("Populating a map of FSN to SCTID for the incoming International release"); + // Convert FSN to a lower case for case-insensitive comparison + incomingData.values().stream() + .filter(HistoricData::isActive) + .forEach(v -> fsnToSctIdMap.put(v.getFsn().toLowerCase(), String.valueOf(v.getConceptId()))); + + LOGGER.info("Looking for FSN duplicates in the incoming International release"); + for (Concept c : gl.getAllConcepts()) { + if (c.isActiveSafely() && inScope(c)) { + lookForDuplicateFsn(c); } } } @@ -601,7 +718,7 @@ protected boolean inScope(Component c) { } private boolean hasTranslation(Concept c) { - return c.getDescriptions().stream() + return c.getDescriptions(ActiveState.ACTIVE).stream() .anyMatch(d -> !d.getLang().equals("en")); } @@ -661,6 +778,8 @@ class TranslationStats { int newConceptCount = 0; int changedFSNCount = 0; int changedFSNCountNoCurrent = 0; + int changedPTCount = 0; + int changedPTCountNoCurrent = 0; int translatedInactivatedCount = 0; int translatedInactivatedWithoutReplacement = 0; } diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java index e0fc871c15..30369952a0 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/HistoricDataUser.java @@ -40,9 +40,7 @@ public class HistoricDataUser extends TermServerReport { public static final boolean DEBUG_TO_FILE = false; - protected String thisRelease; protected String prevRelease; - protected String thisDependency; protected String prevDependency; diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PackageComparisonReport.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PackageComparisonReport.java index 8c8271cca0..60ac33d904 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PackageComparisonReport.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/PackageComparisonReport.java @@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory; import org.snomed.otf.scheduler.domain.*; import org.snomed.otf.scheduler.domain.Job.ProductionStatus; -import org.snomed.otf.script.dao.ReportConfiguration; import java.io.*; import java.nio.charset.StandardCharsets; @@ -136,8 +135,6 @@ public Job getJob() { .add(PREV_RELEASE).withType(JobParameter.Type.BUILD_ARCHIVE) .add(PREV_DEPENDENCY).withType(JobParameter.Type.STRING) .add(MODULES).withType(JobParameter.Type.STRING) - .add(REPORT_OUTPUT_TYPES).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportConfiguration.ReportOutputType.GOOGLE.name()) - .add(REPORT_FORMAT_TYPE).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportConfiguration.ReportFormatType.CSV.name()) .build(); return new Job() diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/RefsetMaintenanceReport.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/RefsetMaintenanceReport.java new file mode 100644 index 0000000000..5f9aa98749 --- /dev/null +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/RefsetMaintenanceReport.java @@ -0,0 +1,182 @@ +package org.ihtsdo.termserver.scripting.reports.release; + +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.lang.StringUtils; +import org.ihtsdo.otf.exception.TermServerScriptException; +import org.ihtsdo.otf.rest.client.terminologyserver.pojo.RefsetMember; +import org.ihtsdo.termserver.scripting.ReportClass; +import org.ihtsdo.termserver.scripting.TermServerScript; +import org.ihtsdo.termserver.scripting.domain.*; +import org.ihtsdo.termserver.scripting.reports.TermServerReport; +import org.ihtsdo.termserver.scripting.util.SnomedUtils; +import org.snomed.otf.scheduler.domain.*; +import org.snomed.otf.scheduler.domain.Job.ProductionStatus; +import org.snomed.otf.scheduler.domain.JobParameter.Type; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Lists active members of simple-type reference sets where the referenced concept is inactive. + * Unlike {@link InactiveConceptInRefset} (which is restricted to concepts inactivated in the + * current authoring cycle), this report includes inactive concepts regardless of when they + * were retired — useful for refset maintainers auditing outdated memberships. + * + *

Default refset scope: descendants of {@code 446609009 |Simple type reference set|}. + * An optional {@code ECL} parameter overrides the default, allowing a narrower scope + * (e.g. a single refset ID, or a curated OR-list).

+ * + *

Refset memberships are read directly from Snowstorm via per-refset GET queries, + * not from {@code Concept.getOtherRefsetMembers()} — the locally-loaded refset graph + * frequently misses members for refsets whose files weren't bundled in the loaded + * archive. Snowstorm gives us the authoritative branch state.

+ */ +public class RefsetMaintenanceReport extends TermServerReport implements ReportClass { + + private static final Logger LOGGER = LoggerFactory.getLogger(RefsetMaintenanceReport.class); + + private static final long REFSET_PAUSE_MS = 200L; + + private static final String DEFAULT_REFSET_ECL = "<446609009 |Simple type reference set|"; + + private String userECL; + private Set targetRefsetIds; + // Cache of refset-id → Concept resolved from Snowstorm, used to fill the Refset FSN + // column when the locally-loaded refset concept is a stub. + private Map targetRefsetsById; + + public static void main(String[] args) throws TermServerScriptException { + TermServerScript.run(RefsetMaintenanceReport.class, args, new HashMap<>()); + } + + @Override + public void init(JobRun run) throws TermServerScriptException { + getArchiveManager().setEnsureSnapshotPlusDeltaLoad(true); + getArchiveManager().setLoadOtherReferenceSets(true); + userECL = run.getParamValue(ECL); + super.init(run); + } + + @Override + public void postInit() throws TermServerScriptException { + String ecl = StringUtils.isEmpty(userECL) ? DEFAULT_REFSET_ECL : userECL; + LOGGER.info("Resolving refsets from ECL: {}", ecl); + Collection refsets = findConcepts(ecl); + targetRefsetsById = refsets.stream() + .collect(Collectors.toMap(Concept::getId, c -> c, (a, b) -> a)); + targetRefsetIds = targetRefsetsById.keySet(); + LOGGER.info("Found {} reference set(s) in scope", targetRefsetIds.size()); + + String[] columnHeadings = new String[] { + "Refset Id, Refset FSN, Concept Id, Concept FSN, SemTag, Reason, Assoc Type, Replacement Id, Replacement FSN" + }; + String[] tabNames = new String[] { "Outdated Memberships" }; + super.postInit(tabNames, columnHeadings); + } + + @Override + public Job getJob() { + JobParameters params = new JobParameters() + .add(ECL).withType(Type.ECL) + .build(); + return new Job() + .withCategory(new JobCategory(JobType.REPORT, JobCategory.RELEASE_STATS)) + .withName("Refset Maintenance Report") + .withDescription("Lists active members of simple-type reference sets whose referenced " + + "concept is inactive, with the inactivation reason and suggested replacement " + + "(historical association). Unlike 'Inactivated Concepts in Refsets', this report " + + "includes inactive concepts from any release cycle, not just the current one. " + + "An optional ECL parameter filters which refsets are included (default: " + + "all simple-type reference sets).") + .withProductionStatus(ProductionStatus.PROD_READY) + .withParameters(params) + .withTag(INT).withTag(MS) + .build(); + } + + @Override + public void runJob() throws TermServerScriptException { + // Set of inactive concept IDs for fast membership checks. isActiveSafely() treats + // null-active as false, so "stub" concepts (referenced by indicator/association + // files but not by any concept-file row) are included — they're genuinely + // inactive from prior release cycles. + Set inactiveConceptIds = gl.getAllConcepts().stream() + .filter(c -> !c.isActiveSafely()) + .map(Concept::getConceptId) + .collect(Collectors.toSet()); + + String branchPath = project.getBranchPath(); + LOGGER.info("Querying Snowstorm for members of {} refsets (filtering {} inactive concepts client-side)", + targetRefsetIds.size(), inactiveConceptIds.size()); + + // Iterate refsets one at a time using GET /members?referenceSet=. This avoids + // POST /members/search (unsupported by the authoring-proxy URL) and keeps each + // request short. tsClient.findRefsetMembers handles pagination internally. + int outdatedMembershipsFound = 0; + int refsetNum = 0; + for (String refsetId : targetRefsetIds) { + refsetNum++; + Collection members = tsClient.findRefsetMembers(branchPath, refsetId, null); + int matchesThisRefset = 0; + for (RefsetMember m : members) { + if (!m.isActiveSafely()) { + continue; + } + if (!inactiveConceptIds.contains(m.getReferencedComponentId())) { + continue; + } + Concept concept = gl.getConcept(m.getReferencedComponentId(), true, false); + reportMember(m, concept); + outdatedMembershipsFound++; + matchesThisRefset++; + } + if (matchesThisRefset > 0 || refsetNum % 20 == 0) { + LOGGER.info("[{}/{}] Refset {}: {} members, {} on inactive concepts (running total: {})", + refsetNum, targetRefsetIds.size(), refsetId, members.size(), + matchesThisRefset, outdatedMembershipsFound); + } + try { + Thread.sleep(REFSET_PAUSE_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + LOGGER.info("Done — {} outdated memberships found across {} refsets", + outdatedMembershipsFound, targetRefsetIds.size()); + } + + private void reportMember(RefsetMember m, Concept concept) throws TermServerScriptException { + // Prefer the Snowstorm-resolved refset concept (has FSN populated as a string) + // over the locally-loaded one, which may be a stub for some extension refsets. + Concept refset = targetRefsetsById.get(m.getRefsetId()); + if (refset == null) { + refset = gl.getConcept(m.getRefsetId()); + } + InactivationIndicator reason = concept.getInactivationIndicator(); + + List assocs = concept.getAssociationEntries(ActiveState.ACTIVE, true); + + if (assocs.isEmpty()) { + report(PRIMARY_REPORT, + refset.getId(), refset.getFsn(), + concept.getId(), concept.getFsn(), concept.getSemTag(), + reason, "N/A", "N/A", "N/A"); + countIssue(concept); + } else { + for (AssociationEntry a : assocs) { + String assocType = SnomedUtils.getAssociationType(a); + Concept target = gl.getConcept(a.getTargetComponentId()); + report(PRIMARY_REPORT, + refset.getId(), refset.getFsn(), + concept.getId(), concept.getFsn(), concept.getSemTag(), + reason, assocType, + target == null ? "" : target.getId(), + target == null ? "" : target.getFsn()); + countIssue(concept); + } + } + } +} diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ReleaseIssuesReport.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ReleaseIssuesReport.java index 282353fc52..29b293ebe9 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ReleaseIssuesReport.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/ReleaseIssuesReport.java @@ -71,13 +71,16 @@ */ public class ReleaseIssuesReport extends TermServerReport implements ReportClass { - record UnwantedChar(String ch, String label, String issueStr) {} + record UnwantedChar(String ch, String label, String issueStr) { + } private static final Logger LOGGER = LoggerFactory.getLogger(ReleaseIssuesReport.class); private static final String CANNOT_READ = "Cannot read "; private static final String FAILURE_WHILE_READING = "Failure while reading: "; private static final String LOADING = "Loading {} ..."; + private static final String ISSUES = "Issues"; + private static final String ITEMS_OF_INTEREST = "Items of Interest"; private static final String FULL_STOP = "."; Set stopWords = new HashSet<>(); @@ -85,7 +88,7 @@ record UnwantedChar(String ch, String label, String issueStr) {} List wordsOftenTypedTwice = new ArrayList<>(); private static final String URL_REGEX = "https?://\\S+\\b"; - + //See https://regex101.com/r/CAlQjx/1/ public static final String SCTID_FSN_REGEX = "(\\d{7,})(\\s+)?\\|(.+?)\\|"; private Pattern sctidFsnPattern; @@ -99,13 +102,12 @@ record UnwantedChar(String ch, String label, String issueStr) {} private List repeatedWordExceptions; Map semTagHierarchyMap = new HashMap<>(); List allConceptsSorted; - + public static final String SCTID_CF_MOD = "11000241103"; //Common French Module public static final String SCTID_CH_MOD = "2011000195101"; //Swiss Module - - private static final int MUT_IDX_ACTIVE = 0; - private static final int MUT_IDX_MODULEID = 1; + private static final int MUT_IDX_MODULEID = 1; + private List expectedExtensionModules = null; private static final int MAX_DESC_LENGTH = 255; @@ -118,7 +120,7 @@ public static void main(String[] args) throws TermServerScriptException { } @Override - public void init (JobRun run) throws TermServerScriptException { + public void init(JobRun run) throws TermServerScriptException { ReportSheetManager.setTargetFolderId("15WXT1kov-SLVi4cvm2TbYJp_vBMr4HZJ"); //Release Validation this.ignoreInputFileForReportName = true; super.init(run); @@ -176,7 +178,7 @@ public void init (JobRun run) throws TermServerScriptException { wordsOftenTypedTwice.add("with"); wordsOftenTypedTwice.add("Be"); wordsOftenTypedTwice.add("be"); - + sctidFsnPattern = Pattern.compile(SCTID_FSN_REGEX, Pattern.MULTILINE); } @@ -227,20 +229,22 @@ private void loadRepeatedWordExceptions() throws TermServerScriptException { @Override public void postInit() throws TermServerScriptException { - String[] columnHeadings = new String[] { + String[] columnHeadings = new String[]{ + "Category, Item, Count", "SCTID, FSN, Semtag, Issue, Legacy, C/D/R Active, Detail, Additional Detail, Further Detail", - "Issue, Count" + "SCTID, FSN, Semtag, Item of Interest, Legacy, C/D/R Active, Detail, Additional Detail, Further Detail", }; - String[] tabNames = new String[] { - "Issues", - "Summary" + String[] tabNames = new String[]{ + "Summary", + ISSUES, + ITEMS_OF_INTEREST }; - + super.postInit(tabNames, columnHeadings); deprecatedHierarchies = new HashSet<>(); deprecatedHierarchies.add(gl.getConcept("116007004|Combined site (body structure)|")); - + if (isMS()) { String defaultModule = project.getMetadata().getDefaultModuleId(); expectedExtensionModules = project.getMetadata().getExpectedExtensionModules(); @@ -249,7 +253,7 @@ public void postInit() throws TermServerScriptException { expectedExtensionModules = Collections.singletonList(defaultModule); } } - + semTagHierarchyMap.put("(regime/therapy)", gl.getConcept("243120004|Regimes and therapies (regime/therapy)|")); } @@ -257,17 +261,17 @@ public void postInit() throws TermServerScriptException { public Job getJob() { JobParameters params = new JobParameters() .add(INCLUDE_ALL_LEGACY_ISSUES) - .withType(JobParameter.Type.BOOLEAN) - .withDefaultValue(false) + .withType(JobParameter.Type.BOOLEAN) + .withDefaultValue(false) .add(UNPROMOTED_CHANGES_ONLY) - .withType(JobParameter.Type.BOOLEAN) - .withDefaultValue(true) + .withType(JobParameter.Type.BOOLEAN) + .withDefaultValue(true) .build(); return new Job() .withCategory(new JobCategory(JobType.REPORT, JobCategory.RELEASE_VALIDATION)) .withName("Release Issues Report") - .withDescription("This report lists a range of potential issues identified in INFRA-2723. " + + .withDescription("This report lists a range of potential issues identified in INFRA-2723. " + "\nThe options that can be selected are:" + "\nUnpromoted: New changes on the branch that have not been promoted yet. Can be task or project." + "\nLegacy: This will include all issues, including old legacy ones - should NOT be combined with Unpromoted." + @@ -301,7 +305,7 @@ public void runJob() throws TermServerScriptException { unexpectedRelationshipModules(); unexpectedAxiomModules(); } - + LOGGER.info("...description rules"); maxLengthCheck(); fullStopInSynonym(); @@ -331,14 +335,14 @@ public void runJob() throws TermServerScriptException { duplicateSemanticTags(); getReportManager().flushFiles(false); - + LOGGER.info("...parent hierarchies (~20 seconds)"); parentsInSameTopLevelHierarchy(); - + LOGGER.info("...axiom integrity"); axiomIntegrity(); noStatedRelationships(); - + LOGGER.info("...Disease semantic tag rule"); diseaseIntegrity(); @@ -349,29 +353,28 @@ public void runJob() throws TermServerScriptException { LOGGER.info("...Text definition dialect checks"); textDefinitionDialectChecks(); } - + LOGGER.info("...Nested brackets check"); nestedBracketCheck(); - + LOGGER.info("...Modelling rules check"); validateAttributeDomainModellingRules(); validateAttributeTypeValueModellingRules(); validateInterpretsHasInterpretation(); neverGroupTogether(); domainMustNotUseType(); - + LOGGER.info("...Deprecation rules"); checkDeprecatedHierarchies(); - + LOGGER.info("...MRCM validation"); checkMRCMDomain(); checkMRCMAttributeRanges(); checkMRCMAttributeDomains(); checkMRCMModuleScope(); - LOGGER.info("Checks complete, creating summary tag"); - populateSummaryTabAndTotal(SECONDARY_REPORT); - + LOGGER.info("Checks complete, populating summary tab"); + reportSummaryCounts(PRIMARY_REPORT, SUMMARY_SORT_ORDER.COUNT); LOGGER.info("Summary tab complete, all done."); } @@ -394,11 +397,11 @@ private void checkComponentsReferenceDependentModules() throws TermServerScriptE continue; } if (!referencedModule.equals(moduleId) && - !gl.getMdrs().getDependencies(moduleId).contains(referencedModule)) { + !gl.getMdrs().getDependencies(moduleId).contains(referencedModule)) { Concept owningConcept = gl.getComponentOwner(c.getId()); String msg = "Component references component in module " + referencedModule + " which is not visible from its own module " + moduleId; - reportAndIncrementSummary(owningConcept, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(c, referencedComponent), msg, c.toString()); + reportAndIncrementCategoryCount(ISSUES, owningConcept, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(c, referencedComponent), msg, c.toString()); } } } @@ -408,6 +411,8 @@ private void checkComponentsReferenceDependentModules() throws TermServerScriptE private void inappropriateModuleJumping() throws TermServerScriptException { String issueStr = "Component module jumped, otherwise unchanged."; String issueStr2 = "Component module jumped without parent"; + String issueStr3 = "Component moved to another module, matching concept module"; + String issueStr4 = "Concept reactivated in another module"; initialiseSummary(issueStr); initialiseSummary(issueStr2); LOGGER.info("Started inappropriateModuleJumping check"); @@ -418,13 +423,13 @@ private void inappropriateModuleJumping() throws TermServerScriptException { if (!c.hasPreviousStateDataRecorded()) { continue; } - + //We'll give inferred relationships the benefit of the doubt //They can be changed by extensions without changing the owning component if (c instanceof Relationship) { continue; } - + String[] previousState = c.getPreviousState(); String[] currentState = c.getMutableFields(); if (previousState.length != currentState.length) { @@ -439,10 +444,10 @@ private void inappropriateModuleJumping() throws TermServerScriptException { if (previousState[MUT_IDX_MODULEID].equals(currentState[MUT_IDX_MODULEID])) { continue; } - + //Check what fields are different. It's a problem if ONLY the moduleId has changed boolean differenceFound = false; - for (int idx=0; idx < previousState.length; idx++) { + for (int idx = 0; idx < previousState.length; idx++) { if (idx == 1) { //If the module hasn't changed, no need to check fields any further if (previousState[idx].equals(currentState[idx])) { @@ -456,33 +461,37 @@ private void inappropriateModuleJumping() throws TermServerScriptException { break; } } - - if (!differenceFound) { - String msg = c.getComponentType() + " previously: " + Arrays.toString(previousState) + " vs current " + Arrays.toString(c.getMutableFields()); - reportAndIncrementSummary(concept, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(concept,c), msg, c, c.getId()); + + //If the component module has jumped but the concept has also, then we could be looking + //at a reactivation in an extension, or still active descriptions might just be taken along + //as part of a transfer of ownership. Log this in a new category + String msg = c.getComponentType() + " previously: " + Arrays.toString(previousState) + " vs current " + Arrays.toString(c.getMutableFields()); + if (!(c instanceof Concept) && concept.getModuleId().equals(c.getModuleId())) { + //The owning concept or description has also jumped module, so we'll flag this up as interesting + reportAndIncrementCategoryCount(ITEMS_OF_INTEREST, concept, isLegacySimple(c), issueStr3, getLegacyIndicator(c), isActive(concept, c), msg, c, c.getId()); + } else if (!differenceFound) { + reportAndIncrementCategoryCount(ISSUES, concept, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(concept, c), msg, c, c.getId()); + } else if (isConceptReactivation(c, previousState)) { + reportAndIncrementCategoryCount(ITEMS_OF_INTEREST, concept, isLegacySimple(c), issueStr4, getLegacyIndicator(c), isActive(concept, c), msg, c, c.getId()); } else { //Now even if there IS a difference, then we don't expect components to change //module without their parent object - concept or description - Component owningObject = SnomedUtils.getParentComponent(c, gl); - if (owningObject == null) { - LOGGER.warn("Could not determine owner of {}", c); - } else if (!hasChangedModule(owningObject)) { - if (owningObject.getModuleId().equals(SCTID_CF_MOD) && - isExpectedModuleJumpException(c, previousState, currentState)) { - continue; - } - - String msg = c.getComponentType() + " previously: " + Arrays.toString(previousState) + " vs current " + Arrays.toString(c.getMutableFields()); - reportAndIncrementSummary(concept, isLegacySimple(c), issueStr2, getLegacyIndicator(c), isActive(concept,c), msg, c, c.getId()); - } + reportAndIncrementCategoryCount(ISSUES, concept, isLegacySimple(c), issueStr2, getLegacyIndicator(c), isActive(concept, c), msg, c, c.getId()); } } } LOGGER.info("Completed inappropriateModuleJumping check"); } + private boolean isConceptReactivation(Component c, String[] previousState) { + return (c instanceof Concept concept + && concept.isActiveSafely() + && previousState != null + && previousState[0].equals("0")); + } + private boolean nullCheck(String view, String[] viewState, Component c) { - for (int i=0; i < viewState.length; i++) { + for (int i = 0; i < viewState.length; i++) { if (viewState[i] == null) { LOGGER.error("Null value at idx {} in {} state of {}", i, view, c); return true; @@ -491,44 +500,13 @@ private boolean nullCheck(String view, String[] viewState, Component c) { return false; } - private boolean isExpectedModuleJumpException(Component c, String[] previousState, String[] currentState) { - String prevModule = previousState[MUT_IDX_MODULEID]; - String currModule = currentState[MUT_IDX_MODULEID]; - String prevActive = previousState[MUT_IDX_ACTIVE]; - - //RP-675 Add allowance for CF LRS entries on CF descriptions being inactivated in CH Module - if (c instanceof LangRefsetEntry && - prevModule.equals(SCTID_CF_MOD) && - currModule.equals(SCTID_CH_MOD) && - !c.isActiveSafely() && prevActive.equals("1")) { - return true; - } - return false; - } - - private boolean hasChangedModule(Component c) throws TermServerScriptException { - //If the component has an effective time, then it hasn't changed in this release - if (!StringUtils.isEmpty(c.getEffectiveTime())) { - return false; - } - String[] previousState = c.getPreviousState(); - String[] currentState = c.getMutableFields(); - if (previousState.length != currentState.length) { - throw new TermServerScriptException("Investigate: component's state has changed length! Previous state: '" + c.getIssues() + "' vs current: " + c); - } - - String prevModule = previousState[MUT_IDX_MODULEID]; - String currModule = currentState[MUT_IDX_MODULEID]; - return prevModule.equals(currModule); - } - //ISRS-286 Ensure Parents in same module. //This check does not apply to MS private void parentsInSameModule() throws TermServerScriptException { if (isMS()) { return; } - + String issueStr = "Mismatching parent moduleId"; initialiseSummary(issueStr); for (Concept c : allActiveConceptsSorted) { @@ -539,16 +517,16 @@ private void parentsInSameModule() throws TermServerScriptException { if (!c.getModuleId().equals(SCTID_CORE_MODULE) && !c.getModuleId().equals(SCTID_MODEL_MODULE)) { continue; } - + //Also skip the top of the metadata hierarchy - it has a core parent //900000000000441003 |SNOMED CT Model Component (metadata)| if (!c.isActiveSafely() || c.getConceptId().equals("900000000000441003")) { continue; } - + for (Concept p : c.getParents(CharacteristicType.STATED_RELATIONSHIP)) { if (!p.getModuleId().equals(c.getModuleId())) { - reportAndIncrementSummary(c, isLegacySimple(c), issueStr,getLegacyIndicator(c), isActive(c,null), p); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(c, null), p); } } } @@ -557,35 +535,35 @@ private void parentsInSameModule() throws TermServerScriptException { //ISRS-391 Descriptions whose module id does not match that of the component //It's OK to add translations to core concepts, so does not apply to MS private void unexpectedDescriptionModules() throws TermServerScriptException { - String issueStr ="Unexpected Description Module"; + String issueStr = "Unexpected Description Module"; initialiseSummary(issueStr); for (Concept c : allConceptsSorted) { for (Description d : c.getDescriptions()) { if (!d.getModuleId().equals(c.getModuleId())) { String msg = "Concept module " + c.getModuleId() + " vs Desc module " + d.getModuleId(); - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c,d), msg, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), msg, d); } } } } - + /* Since and extension is based on a release, any modified description should * belong to the default module */ private void unexpectedComponentModulesMS() throws TermServerScriptException { - String issueStr ="Unexpected module for modified component"; + String issueStr = "Unexpected module for modified component"; initialiseSummary(issueStr); LOGGER.info("Checking {} for unexpected component modules in modified components", allConceptsSorted.size()); for (Concept c : allConceptsSorted) { - for (Component comp: SnomedUtils.getAllComponents(c)) { + for (Component comp : SnomedUtils.getAllComponents(c)) { if (StringUtils.isEmpty(comp.getEffectiveTime()) && !expectedExtensionModules.contains(comp.getModuleId())) { String msg = "Modified component module " + comp.getModuleId() + " is not expected in this extension"; - reportAndIncrementSummary(c, isLegacySimple(comp), issueStr, getLegacyIndicator(comp), isActive(c,comp), msg, comp); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(comp), issueStr, getLegacyIndicator(comp), isActive(c, comp), msg, comp); } } } } - + //ISRS-392 Part II Stated Relationships whose module id does not match that of the component private void unexpectedRelationshipModules() throws TermServerScriptException { String issueStr = "Unexpected Inf Rel Module"; @@ -594,12 +572,12 @@ private void unexpectedRelationshipModules() throws TermServerScriptException { for (Relationship r : c.getRelationships(CharacteristicType.INFERRED_RELATIONSHIP, ActiveState.ACTIVE)) { if (!r.getModuleId().equals(c.getModuleId())) { String msg = "Concept module " + c.getModuleId() + " vs Rel module " + r.getModuleId(); - reportAndIncrementSummary(c, isLegacySimple(r), issueStr, getLegacyIndicator(r), isActive(c,r), msg, r); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(r), issueStr, getLegacyIndicator(r), isActive(c, r), msg, r); } } } } - + private void unexpectedAxiomModules() throws TermServerScriptException { String issueStr = "Unexpected Axiom Module"; initialiseSummary(issueStr); @@ -607,12 +585,12 @@ private void unexpectedAxiomModules() throws TermServerScriptException { for (AxiomEntry a : c.getAxiomEntries()) { if (!a.getModuleId().equals(c.getModuleId())) { String msg = "Concept module " + c.getModuleId() + " vs Axiom module " + a.getModuleId(); - reportAndIncrementSummary(c, isLegacySimple(a),issueStr, getLegacyIndicator(a), isActive(c,a), msg, a); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(a), issueStr, getLegacyIndicator(a), isActive(c, a), msg, a); } } } } - + //MAINT-224 Synonyms created as TextDefinitions new content only private void fullStopInSynonym() throws TermServerScriptException { String issueStr = "Possible TextDefn as Synonym"; @@ -627,18 +605,16 @@ private void fullStopInSynonym() throws TermServerScriptException { //Unless we're interested in legacy issues if (c.isActiveSafely() && (includeLegacyIssues || SnomedUtils.hasNewChanges(c))) { for (Description d : c.getDescriptions(Acceptability.BOTH, DescriptionType.SYNONYM, ActiveState.ACTIVE)) { - if (inScope(d)) { - if (d.getTerm().endsWith(FULL_STOP) && d.getTerm().length() > MIN_TEXT_DEFN_LENGTH) { - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c,d), d); - } + if (inScope(d) && d.getTerm().endsWith(FULL_STOP) && d.getTerm().length() > MIN_TEXT_DEFN_LENGTH) { + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), d); } } - + if (inScope(c)) { //Check we've only got max 1 Text Defn for each dialect if (c.getDescriptions(US_ENG_LANG_REFSET, Acceptability.BOTH, DescriptionType.TEXT_DEFINITION, ActiveState.ACTIVE).size() > 1 || - c.getDescriptions(GB_ENG_LANG_REFSET, Acceptability.BOTH, DescriptionType.TEXT_DEFINITION, ActiveState.ACTIVE).size() > 1 ) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issue2Str, "N", "Y"); + c.getDescriptions(GB_ENG_LANG_REFSET, Acceptability.BOTH, DescriptionType.TEXT_DEFINITION, ActiveState.ACTIVE).size() > 1) { + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issue2Str, "N", "Y"); } } } @@ -667,7 +643,7 @@ private void checkDescriptionsForExcessiveLength(Concept c, String issueStr) thr } if (inScope(d) && d.getTerm().length() > MAX_DESC_LENGTH) { - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c,d), d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), d); } } } @@ -684,35 +660,35 @@ private void missingFSN_PT() throws TermServerScriptException { for (Concept c : allConceptsSorted) { if (inScope(c) && isInternational(c) && (includeLegacyIssues || recentlyTouched.contains(c))) { if (c.getFSNDescription() == null || !c.getFSNDescription().isActiveSafely()) { - reportAndIncrementSummary(c, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(c,null)); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(c), issueStr, getLegacyIndicator(c), isActive(c, null)); } - + Description usPT = c.getPreferredSynonym(US_ENG_LANG_REFSET); if (usPT == null || !usPT.isActiveSafely()) { - reportAndIncrementSummary(c, isLegacySimple(c), issue2Str, getLegacyIndicator(c), isActive(c,null)); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(c), issue2Str, getLegacyIndicator(c), isActive(c, null)); } - + Description gbPT = c.getPreferredSynonym(GB_ENG_LANG_REFSET); if (gbPT == null || !gbPT.isActiveSafely()) { - reportAndIncrementSummary(c, isLegacySimple(c), issue3Str, getLegacyIndicator(c), isActive(c,null)); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(c), issue3Str, getLegacyIndicator(c), isActive(c, null)); } } } } - + private void missingSemanticTag() throws TermServerScriptException { - String issueStr = "Concept (recently touched) with invalid FSN"; + String issueStr = "Concept (recently touched or active) with invalid FSN"; initialiseSummary(issueStr); for (Concept c : allConceptsSorted) { if (inScope(c) - && recentlyTouched.contains(c) - && c.getFsn() != null - && SnomedUtilsBase.deconstructFSN(c.getFsn(), includeLegacyIssues)[1] == null) { - reportAndIncrementSummary(c, false, issueStr, "N", isActive(c,c.getFSNDescription()), c.getFsn()); + && (c.isActiveSafely() || recentlyTouched.contains(c)) + && c.getFsn() != null + && SnomedUtilsBase.deconstructFSN(c.getFsn(), includeLegacyIssues)[1] == null) { + reportAndIncrementCategoryCount(ISSUES, c, false, issueStr, "N", isActive(c, c.getFSNDescription()), c.getFsn()); } } } - + private void semTagInCorrectHierarchy() throws TermServerScriptException { String issueStr = "SemTag used outside of expected hierarchy"; initialiseSummary(issueStr); @@ -722,7 +698,7 @@ private void semTagInCorrectHierarchy() throws TermServerScriptException { String semTag = SnomedUtilsBase.deconstructFSN(c.getFsn(), true)[1]; //Are we in the appropriate Hierarchy? if (semTag != null && semTag.equals(entry.getKey()) && !c.getAncestors(NOT_SET).contains(entry.getValue())) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, "-", isActive(c,c.getFSNDescription()), entry.getValue()); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, "-", isActive(c, c.getFSNDescription()), entry.getValue()); } } } @@ -765,7 +741,7 @@ private boolean repeatedWordGroups(Concept c, Description d, ConcernLevel concer for (int x = 0; x < words.length; x++) { if (stopWords.contains(words[x]) || repeatedWordExceptions.contains(words[x]) || - words[x].length() <= 2 ) { + words[x].length() <= 2) { continue; } @@ -777,7 +753,7 @@ private boolean repeatedWordGroups(Concept c, Description d, ConcernLevel concer //Word 1 will already have been tested against eg word 5, so no need to test word 5 against word 1. //Therefore, start y wherever x is. Plus we don't need to test the same word against itself, so plus 1. for (int y = x + 1; y < words.length; y++) { - if (checkForDuplicatedWordGroups(x, y, c, d, words, concern)) { + if (checkForDuplicatedWordGroups(x, y, c, d, words, concern)) { return true; } } @@ -795,7 +771,7 @@ private boolean checkForSideBySideRepeats(int x, Concept c, Description d, Strin boolean wordsEqual = currentWord.equalsIgnoreCase(nextWord); boolean wordOftenTypedTwice = wordsOftenTypedTwice.contains(currentWord); if (wordsEqual && wordOftenTypedTwice) { - reportAndIncrementSummary(c, isLegacySimple(d), wordIssueStr, getLegacyIndicator(d), isActive(c, d), "Repeated word: " + currentWord, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), wordIssueStr, getLegacyIndicator(d), isActive(c, d), "Repeated word: " + currentWord, d); return true; } } @@ -822,7 +798,7 @@ private boolean checkForDuplicatedWordGroups(int x, int y, Concept c, Descriptio } if (concern.isConcerning(2)) { - reportAndIncrementSummary(c, isLegacySimple(d), wordGroupIssueStr, getLegacyIndicator(d), isActive(c, d), "Repeated word: " + words[x], d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), wordGroupIssueStr, getLegacyIndicator(d), isActive(c, d), "Repeated word: " + words[x], d); return true; } } @@ -846,7 +822,7 @@ private void reviewContractions() throws TermServerScriptException { if ((unpromotedChangesOnly && unpromotedChangesHelper.hasUnpromotedChange(c) && "cannot".equalsIgnoreCase(currentWord)) || "can't".equalsIgnoreCase(currentWord) || (x + 1 < wordsLength && "can".equalsIgnoreCase(currentWord) && "not".equalsIgnoreCase(words[x + 1]))) { - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); continue nextConcept; } } @@ -883,7 +859,7 @@ private void checkForConsecutivePrepositions(Concept c, Description d, String is if (prepositionExceptions.contains(currentWord + " " + nextWord)) { continue; } - reportAndIncrementSummary(c, isLegacySimple(d), issueStr2, getLegacyIndicator(d), isActive(c, d), "Consecutive prepositions: " + currentWord + " " + nextWord, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr2, getLegacyIndicator(d), isActive(c, d), "Consecutive prepositions: " + currentWord + " " + nextWord, d); } } } @@ -896,11 +872,11 @@ private void checkForReversedWords(Concept c, Description d, String issueStr, St int indexOf = wordsOftenTypedInReverse.indexOf(currentWordInReverse); if (indexOf != -1) { String detailStr = String.format("The word '%s' looks to be '%s' in reverse.", currentWord, wordsOftenTypedInReverse.get(indexOf)); - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); } } } - + private void multipleLangRef() throws TermServerScriptException { String issueStr = "Multiple LangRef for a given refset"; initialiseSummary(issueStr); @@ -912,7 +888,7 @@ private void multipleLangRef() throws TermServerScriptException { for (LangRefsetEntry l : d.getLangRefsetEntries(ActiveState.ACTIVE)) { if (activeInRefsets.contains(l.getRefsetId())) { String detailStr = "Description has multiple langrefset entries in same refset: " + l.getRefsetId(); - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); continue nextDescription; } else { activeInRefsets.add(l.getRefsetId()); @@ -922,7 +898,7 @@ private void multipleLangRef() throws TermServerScriptException { } } } - + private void unexpectedLangCodeMS() throws TermServerScriptException { //We need a branch to be able to run this query if (getArchiveManager().isLoadDependencyPlusExtensionArchive()) { @@ -949,7 +925,7 @@ private void checkUnexpectedLangCode(Concept c, String issueStr, Map generateRefsetLangCodeMap() { //First populate en-gb and en-us since we always know about those refsetLangCodeMap.put(US_ENG_LANG_REFSET, "en"); refsetLangCodeMap.put(GB_ENG_LANG_REFSET, "en"); - + //Now the optionalLanguageRefsets are laid out nicely Metadata metadata = project.getMetadata(); refsetLangCodeMap.putAll(metadata.getLangRefsetLangMapping()); @@ -986,7 +962,7 @@ private void multiplePTs() throws TermServerScriptException { if (ptMap.containsKey(l.getRefsetId())) { if (inScopePTDetected) { String detailStr = d + " + " + ptMap.get(l.getRefsetId()); - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr, d); continue nextDescription; } } else { @@ -997,7 +973,7 @@ private void multiplePTs() throws TermServerScriptException { } } } - + private void multipleFSNs() throws TermServerScriptException { String issueStr = "Multiple active FSNs in same language"; initialiseSummary(issueStr); @@ -1013,7 +989,7 @@ private void multipleFSNs() throws TermServerScriptException { if (fsnMap.containsKey(d.getLang())) { if (inScopeFSNDetected) { String detailStr = d + ",\n" + fsnMap.get(d.getLang()); - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), detailStr); } } else { fsnMap.put(d.getLang(), d); @@ -1030,18 +1006,18 @@ private void noCncIndicators() throws TermServerScriptException { if (inScope(d)) { for (InactivationIndicatorEntry i : d.getInactivationIndicatorEntries(ActiveState.ACTIVE)) { if (i.getInactivationReasonId().equals(SCTID_INACT_CONCEPT_NON_CURRENT)) { - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), i); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), i); } } } } } } - + private void dueWithoutTo() throws TermServerScriptException { String issueStr = "'Due' not followed by 'to'"; - String[] acceptableAlternativesPost = new String[] { "date", "mostly to", "either to", "with", "next", "new"}; - String[] acceptableAlternativesPre = new String[] { "claim" }; + String[] acceptableAlternativesPost = new String[]{"date", "mostly to", "either to", "with", "next", "new"}; + String[] acceptableAlternativesPre = new String[]{"claim"}; initialiseSummary(issueStr); for (Concept c : allActiveConceptsSorted) { nextDescription: @@ -1066,15 +1042,15 @@ private void dueWithoutTo() throws TermServerScriptException { break; } } - + for (String acceptableAlt : acceptableAlternativesPre) { if (term.contains(acceptableAlt + " due")) { continue nextDescription; } } - + if (!acceptableAltFound && term.indexOf(" due to", idx) == NOT_FOUND) { - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), d); continue nextDescription; } idx = term.indexOf(" due ", idx + 1); @@ -1083,26 +1059,26 @@ private void dueWithoutTo() throws TermServerScriptException { } } } - + private boolean alsoHasSameWordToLeftOrRight(String[] words, int x, int y) { //So we're looking to see if a word left of X is the same as a word to the left of Y - if (x > 0 && y > 0 && words[x-1].equalsIgnoreCase(words[y-1])) { + if (x > 0 && y > 0 && words[x - 1].equalsIgnoreCase(words[y - 1])) { return true; } - - if (x+1 < words.length && y+1 < words.length && words[x+1].equalsIgnoreCase(words[y+1])) { + + if (x + 1 < words.length && y + 1 < words.length && words[x + 1].equalsIgnoreCase(words[y + 1])) { return true; } - + return false; } private boolean compoundToTheLeftOf(String[] words, int x) { //Is there a 'and' or 'width' to the left of x? - for (int y=0; y < x ; y++) { - if (words[y].equalsIgnoreCase("and") || - words[y].equalsIgnoreCase("with") || - words[y].equalsIgnoreCase("of") || + for (int y = 0; y < x; y++) { + if (words[y].equalsIgnoreCase("and") || + words[y].equalsIgnoreCase("with") || + words[y].equalsIgnoreCase("of") || words[y].equalsIgnoreCase("to")) { return true; } @@ -1126,14 +1102,14 @@ private void duplicateSemanticTags() throws TermServerScriptException { String issue2Str = "Multiple semantic tags"; initialiseSummary(issueStr); initialiseSummary(issue2Str); - + Map knownSemanticTags = new HashMap<>(); Set whiteList = new HashSet<>(); whiteList.add("368847001"); whiteList.add("368812009"); whiteList.add("385238005"); whiteList.add("368808003"); - + //First pass through all active concepts to find semantic tags for (Concept c : allActiveConceptsSorted) { if (c.getFSNDescription() == null) { @@ -1147,15 +1123,15 @@ private void duplicateSemanticTags() throws TermServerScriptException { String semTag = SnomedUtilsBase.deconstructFSN(c.getFsn())[1]; if (StringUtils.isEmpty(semTag)) { String legacy = getLegacyIndicator(c.getFSNDescription()); - reportAndIncrementSummary(c, isLegacySimple(c.getFSNDescription()), issueStr, legacy, isActive(c,c.getFSNDescription()), c.getFsn()); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(c.getFSNDescription()), issueStr, legacy, isActive(c, c.getFSNDescription()), c.getFsn()); } else { knownSemanticTags.put(semTag, c); } } } - + LOGGER.info("Collected {} distinct semantic tags", knownSemanticTags.size()); - + //Second pass to see if we have any of these remaining once //the real semantic tag (last set of brackets) has been removed for (Concept c : allActiveConceptsSorted) { @@ -1172,24 +1148,24 @@ private void duplicateSemanticTags() throws TermServerScriptException { continue; } String legacy = getLegacyIndicator(c.getFSNDescription()); - + //Don't log lack of semantic tag for inactive concepts String termWithoutTag = SnomedUtilsBase.deconstructFSN(c.getFsn(), !c.isActiveSafely())[0]; - + //We can shortcut this if we don't have any brackets here. if (!termWithoutTag.contains("(")) { continue; } for (Map.Entry entry : knownSemanticTags.entrySet()) { if (termWithoutTag.contains(entry.getKey())) { - reportAndIncrementSummary(c, "Y".equals(legacy), issue2Str, legacy, isActive(c,c.getFSNDescription()), c.getFsn(), "Contains semtag: " + entry.getKey() + " identified by " + entry.getValue()); + reportAndIncrementCategoryCount(ISSUES, c, "Y".equals(legacy), issue2Str, legacy, isActive(c, c.getFSNDescription()), c.getFsn(), "Contains semtag: " + entry.getKey() + " identified by " + entry.getValue()); } } } } //ISRS-414 Descriptions which contain a non-breaking space - private void unexpectedCharacters () throws TermServerScriptException { + private void unexpectedCharacters() throws TermServerScriptException { AcceptableCharacterValidator acv = AcceptableCharacterValidator.getInstance(); String validatorIssueStr = "Acceptable character violation"; initialiseSummary(validatorIssueStr); @@ -1210,7 +1186,7 @@ private void unexpectedCharacters () throws TermServerScriptException { int pos = term.indexOf(uc.ch()); if (pos != NOT_SET && !allowableException(c, uc.ch(), term)) { String msg = "At position: " + pos; - reportAndIncrementSummary( + reportAndIncrementCategoryCount(ISSUES, c, "Y".equals(legacy), uc.issueStr(), @@ -1225,7 +1201,7 @@ private void unexpectedCharacters () throws TermServerScriptException { // 2) Acceptable character validation (exactly once per description) for (AcceptableCharacterValidator.ValidationIssue issue : acv.validateString(term)) { - reportAndIncrementSummary( + reportAndIncrementCategoryCount(ISSUES, c, "Y".equals(legacy), validatorIssueStr, @@ -1240,20 +1216,20 @@ private void unexpectedCharacters () throws TermServerScriptException { } private List initialiseUnwantedCharChecks() { - String [][] unwantedChars = new String[][] { - { ZEROSP , "Zero-sized space" }, - { NBSPSTR , "Non-breaking space" }, - { LONG_DASH , "MsWord style dash" }, - { EN_DASH , "EN dash" }, - { EM_DASH , "EM dash" }, - { "--" , "Double dash" }, - { RIGHT_APOS , "Right apostrophe" }, - { LEFT_APOS , "Left apostrophe" }, - { RIGHT_QUOTE , "Right quote" }, - { LEFT_QUOTE , "Left quote" }, - { GRAVE_ACCENT , "Grave accent" }, - { ACUTE_ACCENT , "Acute accent" }, - { SOFT_HYPHEN , "Soft hyphen" } + String[][] unwantedChars = new String[][]{ + {ZEROSP, "Zero-sized space"}, + {NBSPSTR, "Non-breaking space"}, + {LONG_DASH, "MsWord style dash"}, + {EN_DASH, "EN dash"}, + {EM_DASH, "EM dash"}, + {"--", "Double dash"}, + {RIGHT_APOS, "Right apostrophe"}, + {LEFT_APOS, "Left apostrophe"}, + {RIGHT_QUOTE, "Right quote"}, + {LEFT_QUOTE, "Left quote"}, + {GRAVE_ACCENT, "Grave accent"}, + {ACUTE_ACCENT, "Acute accent"}, + {SOFT_HYPHEN, "Soft hyphen"} }; List unwantedCharChecks = Arrays.stream(unwantedChars) @@ -1299,7 +1275,7 @@ private void spaceBracket() throws TermServerScriptException { for (Description d : c.getDescriptions(ActiveState.ACTIVE)) { if (inScope(d)) { if (d.getTerm().contains("( ") || d.getTerm().contains(" )")) { - reportAndIncrementSummary(c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c,d), d); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(d), issueStr, getLegacyIndicator(d), isActive(c, d), d); continue nextConcept; } } @@ -1307,17 +1283,17 @@ private void spaceBracket() throws TermServerScriptException { } } } - + //Active concept parents should not belong to more than one top-level hierarchy – please check NEW and LEGACY content for issues private void parentsInSameTopLevelHierarchy() throws TermServerScriptException { String issueStr = "Parent has multiple top level ancestors"; String issue2Str = "Mixed TopLevel Parents"; initialiseSummary(issueStr); initialiseSummary(issue2Str); - + Set whiteList = new HashSet<>(); - whiteList.add(gl.getConcept("411115002 |Drug-device combination product (product)|")); - + whiteList.add(gl.getConcept("411115002 |Drug-device combination product (product)|")); + nextConcept: for (Concept c : allActiveConceptsSorted) { if (!inScope(c)) { @@ -1329,19 +1305,19 @@ private void parentsInSameTopLevelHierarchy() throws TermServerScriptException { } if (c.isActiveSafely()) { String legacy = getLegacyIndicator(c); - + //Skip root concept - has no highest ancestor if (c.equals(ROOT_CONCEPT)) { continue; } - + //If this concept - or any of its ancestors - are whitelisted, then skip - for (Concept a : gl.getAncestorsCache().getAncestorsOrSelf(c)){ + for (Concept a : gl.getAncestorsCache().getAncestorsOrSelf(c)) { if (whiteList.contains(a)) { continue nextConcept; } } - + Concept lastTopLevel = null; for (Concept p : c.getParents(CharacteristicType.INFERRED_RELATIONSHIP)) { //If we are a top level, skip also @@ -1350,27 +1326,27 @@ private void parentsInSameTopLevelHierarchy() throws TermServerScriptException { } //What top level hierarchy is this parent in? Set topLevels = SnomedUtils.getHighestAncestorsBefore(p, ROOT_CONCEPT); - + if (topLevels.size() > 1) { String topLevelStr = topLevels.stream().map(Object::toString).collect(Collectors.joining(",\n")); - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c,null), topLevelStr); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null), topLevelStr); continue nextConcept; } else if (topLevels.isEmpty()) { - reportAndIncrementSummary(c, false, "Failed to find top level of parent ", legacy, isActive(c,null), p); + reportAndIncrementCategoryCount(ISSUES, c, false, "Failed to find top level of parent ", legacy, isActive(c, null), p); continue nextConcept; } - + Concept thisTopLevel = topLevels.iterator().next(); if (lastTopLevel == null) { lastTopLevel = thisTopLevel; - } else if ( !lastTopLevel.equals(thisTopLevel)) { - reportAndIncrementSummary(c, "Y".equals(legacy), issue2Str, legacy, isActive(c,null), thisTopLevel, lastTopLevel); + } else if (!lastTopLevel.equals(thisTopLevel)) { + reportAndIncrementCategoryCount(ISSUES, c, "Y".equals(legacy), issue2Str, legacy, isActive(c, null), thisTopLevel, lastTopLevel); } } } } } - + //RP-128 private void axiomIntegrity() throws TermServerScriptException { String issueStr = "Axiom contains inactive type"; @@ -1381,7 +1357,7 @@ private void axiomIntegrity() throws TermServerScriptException { initialiseSummary(issue2Str); initialiseSummary(issue3Str); initialiseSummary(issue4Str); - + //Check all concepts referenced in relationships are valid for (Concept c : allActiveConceptsSorted) { if (c.isActiveSafely() && inScope(c)) { @@ -1389,13 +1365,13 @@ private void axiomIntegrity() throws TermServerScriptException { for (Relationship r : c.getRelationships(CharacteristicType.STATED_RELATIONSHIP, ActiveState.ACTIVE)) { String legacy = getLegacyIndicator(r); if (!r.getType().isActiveSafely()) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c,r), r); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, r), r); } if (r.isNotConcrete() && !r.getTarget().isActiveSafely()) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issue2Str, legacy, isActive(c,r), r); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issue2Str, legacy, isActive(c, r), r); } } - + //Check all LHS relationships are active for (AxiomEntry a : c.getAxiomEntries(ActiveState.ACTIVE, true)) { try { @@ -1405,52 +1381,53 @@ private void axiomIntegrity() throws TermServerScriptException { if (axiom == null) { continue; } - + for (Relationship r : AxiomUtils.getLHSRelationships(c, axiom)) { if (!r.getType().isActiveSafely()) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issue3Str, legacy, isActive(c,r), r); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issue3Str, legacy, isActive(c, r), r); } if (r.isNotConcrete() && !r.getTarget().isActiveSafely()) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issue4Str, legacy, isActive(c,r), r); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issue4Str, legacy, isActive(c, r), r); } } } catch (ConversionException e) { - LOGGER.error ("Failed to convert: " + a, e); + LOGGER.error("Failed to convert: {}", a, e); } } } } } - + /** * This will not spot many stated relationships because the axiom equivalents * will override these rows. + * * @throws TermServerScriptException */ private void noStatedRelationships() throws TermServerScriptException { String issueStr = "Active stated relationship"; initialiseSummary(issueStr); - + //Check no active relationship is non-axiom for (Concept c : allActiveConceptsSorted) { if (c.isActiveSafely() && inScope(c)) { for (Relationship r : c.getRelationships(CharacteristicType.STATED_RELATIONSHIP, ActiveState.ACTIVE)) { String legacy = getLegacyIndicator(r); if (!r.fromAxiom()) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c,r), r); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, r), r); } } } } } - + //RP-127 private void diseaseIntegrity() throws TermServerScriptException { String issueStr = "Clinical finding has disorder as ancestor "; String issue2Str = "Disorder is not descendant of 64572001|Disease (disorder)| "; initialiseSummary(issueStr); initialiseSummary(issue2Str); - + //Rule 1 (clinical finding) concepts cannot have a (disorder) concept as a parent //Rule 2 All (disorder) concepts must be a descendant of 64572001|Disease (disorder)| Set diseases = DISEASE.getDescendants(NOT_SET); @@ -1462,19 +1439,20 @@ private void diseaseIntegrity() throws TermServerScriptException { if (semTag.equals("(finding)")) { checkForAncestorSemTag(c, issueStr); } else if (semTag.equals("(disorder)") && !diseases.contains(c)) { - String legacy = getLegacyIndicator(c); - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issue2Str, legacy, isActive(c,null)); + boolean isLegacy = !recentlyTouched.contains(c); + String legacyIndicator = isLegacy ? "Y" : "N"; + reportAndIncrementCategoryCount(ISSUES, c, isLegacy, issue2Str, legacyIndicator, isActive(c, null)); } } } - + private void checkForAncestorSemTag(Concept c, String issueStr) throws TermServerScriptException { Set ancestors = c.getAncestors(NOT_SET); for (Concept ancestor : ancestors) { String semTag = SnomedUtilsBase.deconstructFSN(ancestor.getFsn())[1]; if (semTag.equals("(disorder)")) { String legacy = getLegacyIndicator(c); - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c,null), ancestor); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null), ancestor); return; } } @@ -1491,22 +1469,22 @@ private void descriptionDialectChecks() throws TermServerScriptException { Description fsn = c.getFSNDescription(LANG_EN); if (dialectChecker.containsGBSpecificTerm(fsn.getTerm())) { String legacy = getLegacyIndicator(c); - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null), fsn); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null), fsn); } } } - + //RP-165 private void textDefinitionDialectChecks() throws TermServerScriptException { String issueStr = "Text Definition exists in one dialect and not the other"; initialiseSummary(issueStr); - + List bothDialectTextDefns = new ArrayList<>(); for (Concept c : allActiveConceptsSorted) { if (c.isActiveSafely()) { List textDefns = c.getDescriptions(Acceptability.BOTH, DescriptionType.TEXT_DEFINITION, ActiveState.ACTIVE); if (textDefns.size() > 2) { - LOGGER.warn ("{} has {} active text definitions - check for compatibility", c, textDefns.size()); + LOGGER.warn("{} has {} active text definitions - check for compatibility", c, textDefns.size()); } boolean hasUS = false; boolean hasGB = false; @@ -1530,7 +1508,7 @@ private void textDefinitionDialectChecks() throws TermServerScriptException { } if ((hasUS && !hasGB) || (hasGB && !hasUS)) { String legacy = getLegacyIndicator(c); - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c,null)); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null)); } } } @@ -1544,7 +1522,7 @@ private void checkForUsGbSpecificSpelling(List bothDialectTextDefns initialiseSummary(issue2Str); DialectChecker dialectChecker = DialectChecker.create(); LOGGER.debug("Checking {} both-dialect text definitions against {} dialect pairs", bothDialectTextDefns.size(), dialectChecker.size()); - + nextDescription: for (Description textDefn : bothDialectTextDefns) { String term = " " + textDefn.getTerm().toLowerCase().replaceAll("[^A-Za-z0-9]", " "); @@ -1552,7 +1530,7 @@ private void checkForUsGbSpecificSpelling(List bothDialectTextDefns String legacy = getLegacyIndicator(c); for (DialectChecker.DialectPair dialectPair : dialectChecker.getDialectPairs()) { if (checkDialectPair(c, dialectPair.usTermPadded, term, textDefn, issueStr, legacy) || - checkDialectPair(c, dialectPair.gbTermPadded, term, textDefn, issue2Str, legacy)) { + checkDialectPair(c, dialectPair.gbTermPadded, term, textDefn, issue2Str, legacy)) { continue nextDescription; } } @@ -1567,7 +1545,7 @@ private boolean checkDialectPair(Concept c, String dialectSpecificTerm, String t .replaceAll(URL_REGEX, "") .replaceAll("[^A-Za-z0-9]", " "); if (termFiltered.contains(dialectSpecificTerm)) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null), dialectSpecificTerm, textDefn); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, legacy, isActive(c, null), dialectSpecificTerm, textDefn); reported = true; } } @@ -1577,9 +1555,9 @@ private boolean checkDialectPair(Concept c, String dialectSpecificTerm, String t private void nestedBracketCheck() throws TermServerScriptException { String issueStr = "Active description on inactive concept contains nested brackets"; initialiseSummary(issueStr); - Character[][] bracketPairs = new Character[][] {{'(', ')'}, - {'[',']'}}; - + Character[][] bracketPairs = new Character[][]{{'(', ')'}, + {'[', ']'}}; + nextConcept: for (Concept c : allActiveConceptsSorted) { if (!c.isActiveSafely()) { @@ -1587,7 +1565,7 @@ private void nestedBracketCheck() throws TermServerScriptException { if (inScope(d)) { for (Character[] bracketPair : bracketPairs) { if (containsNestedBracket(c, d, bracketPair)) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c,d), d); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, d), d); continue nextConcept; } } @@ -1596,10 +1574,10 @@ private void nestedBracketCheck() throws TermServerScriptException { } } } - + private boolean containsNestedBracket(Concept c, Description d, Character[] bracketPair) throws TermServerScriptException { Stack brackets = new Stack<>(); - for (Character ch: d.getTerm().toCharArray()) { + for (Character ch : d.getTerm().toCharArray()) { if (ch.equals(bracketPair[0])) { //Opening bracket brackets.push(ch); if (brackets.size() > 1) { @@ -1607,7 +1585,7 @@ private boolean containsNestedBracket(Concept c, Description d, Character[] brac } } else if (ch.equals(bracketPair[1])) { //Closing bracket if (brackets.isEmpty()) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), "Closing bracket found without matching opening", getLegacyIndicator(c), isActive(c,d), d); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), "Closing bracket found without matching opening", getLegacyIndicator(c), isActive(c, d), d); } else { brackets.pop(); } @@ -1615,7 +1593,7 @@ private boolean containsNestedBracket(Concept c, Description d, Character[] brac } return false; } - + private void validateAttributeDomainModellingRules() throws TermServerScriptException { //RP-179 concepts using surgical approach must be surgical procedures @@ -1632,27 +1610,28 @@ private void validateAttributeDomainModellingRules() throws TermServerScriptExce } /** - * Where a concept uses the specified attribute type in its modelling, + * Where a concept uses the specified attribute type in its modelling, * ensure that it is a descendant of the specified subhierarchy - * @throws TermServerScriptException + * + * @throws TermServerScriptException */ private void validateTypeUsedInDomain(Concept c, Concept type, Set subHierarchyList, String issueStr) throws TermServerScriptException { if (SnomedUtils.hasType(CharacteristicType.INFERRED_RELATIONSHIP, c, type) && !subHierarchyList.contains(c)) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null)); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null)); } } private void validateAttributeTypeValueModellingRules() throws TermServerScriptException { String issueStr = "Finding/Procedure site cannot take a combined site value"; initialiseSummary(issueStr); - + //RP-181 No finding or procedure site attribute should take a combined bodysite as the value List typesOfInterest = new ArrayList<>(); typesOfInterest.add(FINDING_SITE); Set procSiteTypes = cache.getDescendantsOrSelf(gl.getConcept("363704007 |Procedure site (attribute)|")); typesOfInterest.addAll(procSiteTypes); Set invalidValues = cache.getDescendantsOrSelf(gl.getConcept("116007004 |Combined site (body structure)|")); - + for (Concept c : allActiveConceptsSorted) { if (c.isActiveSafely() && inScope(c)) { for (Concept type : typesOfInterest) { @@ -1693,7 +1672,7 @@ private boolean validateRelationshipGroupForInterpretsHasInterpreation(Concept c breaksCardinalityRules = true; } hasInterprets = true; - } else if ( r.getType().equals(HAS_INTERPRETATION)) { + } else if (r.getType().equals(HAS_INTERPRETATION)) { if (hasHasInterpretation) { breaksCardinalityRules = true; } @@ -1703,11 +1682,11 @@ private boolean validateRelationshipGroupForInterpretsHasInterpreation(Concept c } } if ((hasInterprets || hasHasInterpretation) && hasOtherAttribute) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null), g); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null), g); reported = true; } if (breaksCardinalityRules) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr2, getLegacyIndicator(c), isActive(c, null), g); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr2, getLegacyIndicator(c), isActive(c, null), g); reported = true; } return reported; @@ -1716,12 +1695,12 @@ private boolean validateRelationshipGroupForInterpretsHasInterpreation(Concept c private void checkDeprecatedHierarchies() throws TermServerScriptException { String issueStr = "New concept created in deprecated hierarchy"; initialiseSummary(issueStr); - + //RP-181 No new combined bodysite concepts should be created for (Concept deprecatedHierarchy : deprecatedHierarchies) { for (Concept c : deprecatedHierarchy.getDescendants(NOT_SET)) { if (!c.isReleasedSafely() && inScope(c)) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null), deprecatedHierarchy); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null), deprecatedHierarchy); } } } @@ -1753,7 +1732,7 @@ private void checkMRCMModuleScope() throws TermServerScriptException { checkMRCMTerms("MRCM Module Scope", gl.getMRCMModuleScopeManager().getMrcmModuleScopeMap().get(module)); } } - + private void checkMRCMTerms(String partName, Collection refsetMembers) throws TermServerScriptException { for (RefsetMember rm : refsetMembers) { if (rm.isActiveSafely()) { @@ -1770,40 +1749,40 @@ private void validateTermsInField(String partName, Concept c, RefsetMember rm, S String issueStr2 = partName + " refset field " + fieldName + " contains out of date FSN"; String issueStr3 = partName + " refset field " + fieldName + " is malformed"; String issueStr4 = partName + " refset field " + fieldName + " concept missing preferred term"; - + initialiseSummary(issueStr); initialiseSummary(issueStr2); initialiseSummary(issueStr3); initialiseSummary(issueStr4); - + //Is this field all numeric? Check concept exists if so String field = rm.getField(fieldName); if (org.ihtsdo.otf.utils.StringUtils.isNumeric(field) && field.length() > 7) { Concept refConcept = gl.getConcept(field, false, false); if (refConcept == null || !refConcept.isActiveSafely()) { - reportAndIncrementSummary(c, isLegacySimple(rm), issueStr, getLegacyIndicator(c), isActive(c, null), field, rm.getId(), field); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(rm), issueStr, getLegacyIndicator(c), isActive(c, null), field, rm.getId(), field); } return; } - + Matcher matcher = sctidFsnPattern.matcher(field); while (matcher.find()) { //Group 1 is the SCTID, group 3 is the FSN. Group 2 is optional whitespace if (matcher.groupCount() == 3) { Concept refConcept = gl.getConcept(matcher.group(1), false, false); if (refConcept == null || !refConcept.isActiveSafely()) { - reportAndIncrementSummary(c, isLegacySimple(rm), issueStr, getLegacyIndicator(c), isActive(c, null), refConcept == null ? matcher.group(1) : refConcept, rm.getId(), field); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(rm), issueStr, getLegacyIndicator(c), isActive(c, null), refConcept == null ? matcher.group(1) : refConcept, rm.getId(), field); } else { if (refConcept.getPreferredSynonym(US_ENG_LANG_REFSET) == null) { - reportAndIncrementSummary(c, isLegacySimple(rm), issueStr4, getLegacyIndicator(c), isActive(c, null), refConcept, rm.getId(), field); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(rm), issueStr4, getLegacyIndicator(c), isActive(c, null), refConcept, rm.getId(), field); } else { String fsn = matcher.group(3); if (fsn == null) { - reportAndIncrementSummary(c, isLegacySimple(rm), issueStr3, getLegacyIndicator(c), isActive(c, null), refConcept, rm.getId(), field); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(rm), issueStr3, getLegacyIndicator(c), isActive(c, null), refConcept, rm.getId(), field); } else if (!refConcept.getFsn().equals(fsn)) { //Sometimes we use the PT. Check on the rules for when we use each one. if (!fsn.equals(refConcept.getPreferredSynonym(US_ENG_LANG_REFSET).getTerm())) { - reportAndIncrementSummary(c, isLegacySimple(rm), issueStr2, getLegacyIndicator(c), isActive(c, null), refConcept, "Text in refset field: " + fsn, rm.getId(), field); + reportAndIncrementCategoryCount(ISSUES, c, isLegacySimple(rm), issueStr2, getLegacyIndicator(c), isActive(c, null), refConcept, "Text in refset field: " + fsn, rm.getId(), field); } } } @@ -1815,36 +1794,37 @@ private void validateTermsInField(String partName, Concept c, RefsetMember rm, S /** * If the given concept uses the particular type, checks if that type is in (or must not be in) * the list of specified values - * @throws TermServerScriptException + * + * @throws TermServerScriptException */ private void validateTypeValueCombo(Concept c, Concept type, Set values, String issueStr, - boolean mustBeIn) throws TermServerScriptException { + boolean mustBeIn) throws TermServerScriptException { Set relsWithType = c.getRelationships(CharacteristicType.INFERRED_RELATIONSHIP, type, ActiveState.ACTIVE); for (Relationship relWithType : relsWithType) { //Must the value be in, or must the value be NOT in our list of values? boolean isIn = values.contains(relWithType.getTarget()); if (!isIn == mustBeIn) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(relWithType), isActive(c, relWithType), relWithType); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(relWithType), isActive(c, relWithType), relWithType); } } } - - + + //RP-180 private void neverGroupTogether() throws TermServerScriptException { - Concept[][] neverTogetherList = new Concept[][] + Concept[][] neverTogetherList = new Concept[][] { - { gl.getConcept("363589002 |Associated procedure|"), gl.getConcept("408729009 |Finding context|")}, - { gl.getConcept("408730004 |Procedure context|"), gl.getConcept("246090004 |Associated finding|")} + {gl.getConcept("363589002 |Associated procedure|"), gl.getConcept("408729009 |Finding context|")}, + {gl.getConcept("408730004 |Procedure context|"), gl.getConcept("246090004 |Associated finding|")} }; - + for (Concept[] neverTogether : neverTogetherList) { String issueStr = "Attributes " + neverTogether[0].toStringPref() + " and " + neverTogether[1].toStringPref() + " must not appear in same group"; initialiseSummary(issueStr); for (Concept c : allActiveConceptsSorted) { if (c.isActiveSafely() && inScope(c)) { if (appearInSameGroup(c, neverTogether[0], neverTogether[1])) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null)); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null)); } } } @@ -1854,10 +1834,10 @@ private void neverGroupTogether() throws TermServerScriptException { //RP-180 private void domainMustNotUseType() throws TermServerScriptException { //FORMAT 0 - Domain 1 - Disallowed Attribute 2 - Unless also a member of domain - Concept[][] domainTypeIncompatibilities = new Concept[][] + Concept[][] domainTypeIncompatibilities = new Concept[][] { - { gl.getConcept("413350009 |Finding with explicit context|"), gl.getConcept("363589002 |Associated procedure|"), gl.getConcept("129125009 |Procedure with explicit context|")}, - { gl.getConcept("129125009 |Procedure with explicit context|"), gl.getConcept("408729009 |Finding context|"), gl.getConcept("413350009 |Finding with explicit context|")} + {gl.getConcept("413350009 |Finding with explicit context|"), gl.getConcept("363589002 |Associated procedure|"), gl.getConcept("129125009 |Procedure with explicit context|")}, + {gl.getConcept("129125009 |Procedure with explicit context|"), gl.getConcept("408729009 |Finding context|"), gl.getConcept("413350009 |Finding with explicit context|")} }; for (Concept[] domainType : domainTypeIncompatibilities) { String issueStr = "Domain " + domainType[0] + " should not use attribute type: " + domainType[1]; @@ -1868,13 +1848,13 @@ private void domainMustNotUseType() throws TermServerScriptException { //RP-574 But is this concept also a type of a domain that would allow this attribute? Set ancestors = c.getAncestors(NOT_SET); if (!ancestors.contains(domainType[2])) { - reportAndIncrementSummary(c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null)); + reportAndIncrementCategoryCount(ISSUES, c, !recentlyTouched.contains(c), issueStr, getLegacyIndicator(c), isActive(c, null)); } } } } } - + } private boolean appearInSameGroup(Concept c, Concept c1, Concept c2) { @@ -1897,7 +1877,6 @@ private boolean appearInSameGroup(Concept c, Concept c1, Concept c2) { } - class ConcernLevel { //Need an object to wrap an integer so we can pass it into a function multiple times private int concern = 0; @@ -1927,4 +1906,14 @@ public boolean isConcerning(int threshold) { } } + @Override + protected void initialiseSummary(String issue) { + initialiseSummaryCount(ISSUES, issue); + } + + protected void reportAndIncrementCategoryCount(String category, Concept c, boolean isLegacy, Object... details) throws TermServerScriptException { + boolean isIssueToCount = category.equals(ISSUES); + int tabIdx = isIssueToCount ? SECONDARY_REPORT : TERTIARY_REPORT; + super.reportAndIncrementCategoryCount(tabIdx, category, isIssueToCount, c, isLegacy, details); + } } diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java index e5be956eca..1b1c343f3c 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStats.java @@ -14,8 +14,6 @@ import org.slf4j.LoggerFactory; import org.snomed.otf.scheduler.domain.*; import org.snomed.otf.scheduler.domain.Job.ProductionStatus; -import org.snomed.otf.script.dao.ReportConfiguration.ReportFormatType; -import org.snomed.otf.script.dao.ReportConfiguration.ReportOutputType; import org.snomed.otf.script.dao.ReportSheetManager; import java.io.File; @@ -36,20 +34,42 @@ public class SummaryComponentStats extends HistoricDataUser implements ReportCla static final int MAX_REPORT_TABS = 16; static final int MAX_REPORT_TABS_WITH_HIERARCHY = MAX_REPORT_TABS - 3; - static final int DATA_WIDTH = 29; - - static final int TAB_CONCEPTS = 0, TAB_DESCS = 1, TAB_RELS = 2, TAB_CD = 3, TAB_AXIOMS = 4, - TAB_LANG = 5, TAB_INACT_IND = 6, TAB_HIST = 7, TAB_TEXT_DEFN = 8, TAB_QI = 9, - TAB_DESC_HIST = 10, TAB_DESC_CNC = 11, TAB_DESC_INACT = 12, TAB_REFSET = 13, - TAB_DESC_BY_LANG = 14, TAB_INACT_REASON = 15; - - static final int IDX_NEW = 0, IDX_CHANGED = 1, IDX_INACTIVATED = 2, IDX_REACTIVATED = 3, IDX_NEW_INACTIVE = 4, IDX_NEW_NEW = 5, - IDX_MOVED_MODULE = 6, IDX_CHANGED_INACTIVE = 7, IDX_NEW_P = 8, IDX_NEW_SD = 9, - IDX_TOTAL = 10, IDX_INACT_AMBIGUOUS = 11, IDX_INACT_MOVED_ELSEWHERE = 12, IDX_INACT_CONCEPT_NON_CURRENT = 13, - IDX_INACT_DUPLICATE = 14, IDX_INACT_ERRONEOUS = 15, IDX_INACT_INAPPROPRIATE = 16, IDX_INACT_LIMITED = 17, - IDX_INACT_OUTDATED = 18, IDX_INACT_PENDING_MOVE = 19, IDX_INACT_NON_CONFORMANCE = 20, - IDX_INACT_NOT_EQUIVALENT = 21, IDX_CONCEPTS_AFFECTED = 22, IDX_TOTAL_ACTIVE = 23, IDX_PROMOTED=24, - IDX_NEW_IN_QI_SCOPE = 25, IDX_GAINED_ATTRIBUTES = 26, IDX_LOST_ATTRIBUTES = 27, IDX_INACT_OTHER = 28; + static final int DATA_WIDTH = 17; + + static final int TAB_CONCEPTS = 0; + static final int TAB_DESCS = 1; + static final int TAB_RELS = 2; + static final int TAB_CD = 3; + static final int TAB_AXIOMS = 4; + static final int TAB_LANG = 5; + static final int TAB_INACT_IND = 6; + static final int TAB_HIST = 7; + static final int TAB_TEXT_DEFN = 8; + static final int TAB_QI = 9; + static final int TAB_DESC_HIST = 10; + static final int TAB_DESC_CNC = 11; + static final int TAB_DESC_INACT = 12; + static final int TAB_REFSET = 13; + static final int TAB_DESC_BY_LANG = 14; + static final int TAB_INACT_REASON = 15; + + static final int IDX_NEW = 0; + static final int IDX_CHANGED = 1; + static final int IDX_INACTIVATED = 2; + static final int IDX_REACTIVATED = 3; + static final int IDX_NEW_INACTIVE = 4; + static final int IDX_NEW_NEW = 5; + static final int IDX_MOVED_MODULE = 6; + static final int IDX_CHANGED_INACTIVE = 7; + static final int IDX_NEW_P = 8; + static final int IDX_NEW_SD = 9; + static final int IDX_TOTAL = 10; + static final int IDX_CONCEPTS_AFFECTED = 11; + static final int IDX_TOTAL_ACTIVE = 12; + static final int IDX_PROMOTED = 13; + static final int IDX_NEW_IN_QI_SCOPE = 14; + static final int IDX_GAINED_ATTRIBUTES = 15; + static final int IDX_LOST_ATTRIBUTES = 16; static Map> sheetFieldsByIndex = getSheetFieldsMap(); @@ -111,11 +131,11 @@ public class SummaryComponentStats extends HistoricDataUser implements ReportCla // * Desc Inact "Sctid, Hierarchy, SemTag, New, Changed, Inactivated, Reactivated, New Inactive, New with New Concept, Changed Inactive, Total Active, Total", // * Refsets - "Sctid, Hierarchy, SemTag, New, Changed, Inactivated, Reactivated, New Inactive, New with New Concept, Changed Inactive, Total Active, Total", + "Sctid, Reference Set, SemTag, New, Changed, Inactivated, Reactivated, New Inactive, New with New Concept, Changed Inactive, Total Active, Total", // * Desc By Lang ", Description Type, Language, New, Changed, Inactivated, Reactivated, New Inactive, New with New Concept, Changed Inactive, Total Active, Total", // * Inact Reason - "Sctid, Hierarchy, SemTag, New, Changed, Inactivated, Reactivated, New Inactive, New with New Concept, Changed Inactive, Total Active, Total" + "Sctid, Inactivation Reason, SemTag, New, Changed, Inactivated, Reactivated, New Inactive, New with New Concept, Changed Inactive, Total Active, Total" }; private final String[] tabNames = new String[] { "Concepts", @@ -150,8 +170,6 @@ public static void main(String[] args) throws TermServerScriptException { Map params = new HashMap<>(); params.put(THIS_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20250101T120000Z.zip"); params.put(PREV_RELEASE, "SnomedCT_InternationalRF2_PRODUCTION_20241201T120000Z.zip"); - params.put(REPORT_OUTPUT_TYPES, "S3"); - params.put(REPORT_FORMAT_TYPE, "JSON"); TermServerScript.run(SummaryComponentStats.class, args, params); } @@ -161,8 +179,6 @@ public Job getJob() { .add(THIS_RELEASE).withType(JobParameter.Type.RELEASE_ARCHIVE) .add(PREV_RELEASE).withType(JobParameter.Type.RELEASE_ARCHIVE) .add(MODULES).withType(JobParameter.Type.STRING) - .add(REPORT_OUTPUT_TYPES).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportOutputType.GOOGLE.name()) - .add(REPORT_FORMAT_TYPE).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportFormatType.CSV.name()) .build(); return new Job() @@ -685,8 +701,10 @@ private void outputResults() throws TermServerScriptException { languageSubTotals = outputRefsetData(TAB_REFSET, "language", totals); indicatorSubTotals = outputRefsetData(TAB_REFSET, "indicator", totals); + //Inactivation reason data is not broken down by major hierarchy outputInactivationReasonData(TAB_INACT_REASON, totals); - + + //Description by language data is not broken down by major hierarchy outputDescriptionByLanguage(TAB_DESC_BY_LANG, totals); for (int idxTab = 0; idxTab < MAX_REPORT_TABS; idxTab++) { diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java index 294314e683..384d5e84ce 100644 --- a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/SummaryComponentStatsExtensions.java @@ -7,7 +7,6 @@ import org.ihtsdo.termserver.scripting.TermServerScript; import org.snomed.otf.scheduler.domain.*; import org.snomed.otf.scheduler.domain.Job.ProductionStatus; -import org.snomed.otf.script.dao.ReportConfiguration.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,8 +28,6 @@ public static void main(String[] args) throws TermServerScriptException{ params.put(PREV_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20241001T120000Z.zip"); params.put(MODULES, "57091000202101,51000202101,57101000202106"); //NZ Module - //REPORT_OUTPUT_TYPES, "S3" - //REPORT_FORMAT_TYPE, "JSON" TermServerScript.run(SummaryComponentStatsExtensions.class, args, params); } @@ -43,8 +40,6 @@ public Job getJob() { .add(PREV_RELEASE).withType(JobParameter.Type.RELEASE_ARCHIVE) .add(PREV_DEPENDENCY).withType(JobParameter.Type.STRING) .add(MODULES).withType(JobParameter.Type.STRING) - .add(REPORT_OUTPUT_TYPES).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportOutputType.GOOGLE.name()) - .add(REPORT_FORMAT_TYPE).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportFormatType.CSV.name()) .build(); return new Job() diff --git a/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/browser/BrowserReleaseStatsGenerator.java b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/browser/BrowserReleaseStatsGenerator.java new file mode 100644 index 0000000000..95ee006976 --- /dev/null +++ b/reporting-engine-worker/src/main/java/org/ihtsdo/termserver/scripting/reports/release/browser/BrowserReleaseStatsGenerator.java @@ -0,0 +1,78 @@ +package org.ihtsdo.termserver.scripting.reports.release.browser; + +import org.ihtsdo.otf.exception.TermServerScriptException; +import org.ihtsdo.otf.utils.StringUtils; +import org.ihtsdo.termserver.scripting.TermServerScript; +import org.ihtsdo.termserver.scripting.reports.release.SummaryComponentStats; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.snomed.otf.scheduler.domain.*; +import org.snomed.otf.script.dao.ReportConfiguration; + +import java.util.HashMap; +import java.util.Map; + +@SuppressWarnings("java:S110") +public class BrowserReleaseStatsGenerator extends SummaryComponentStats { + + private static final Logger LOGGER = LoggerFactory.getLogger(BrowserReleaseStatsGenerator.class); + + public static void main(String[] args) throws TermServerScriptException { + Map params = new HashMap<>(); + + params.put(THIS_RELEASE, "SnomedCT_ManagedServiceIE_PRODUCTION_IE1000220_20260321T120000Z.zip"); + params.put(THIS_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20260301T120000Z.zip"); + params.put(PREV_RELEASE, "SnomedCT_ManagedServiceIE_PRODUCTION_IE1000220_20260221T120000Z.zip"); + params.put(PREV_DEPENDENCY, "SnomedCT_InternationalRF2_PRODUCTION_20260201T120000Z.zip"); + params.put(MODULES, "11000220105,1601000220105"); + + params.put(REPORT_OUTPUT_TYPES, "S3"); + params.put(REPORT_FORMAT_TYPE, "JSON"); + TermServerScript.run(BrowserReleaseStatsGenerator.class, args, params); + } + + @Override + public Job getJob() { + JobParameters params = new JobParameters() + .add(THIS_RELEASE).withType(JobParameter.Type.RELEASE_ARCHIVE).withMandatory() + .add(THIS_DEPENDENCY).withType(JobParameter.Type.STRING) + .add(PREV_RELEASE).withType(JobParameter.Type.RELEASE_ARCHIVE).withMandatory() + .add(PREV_DEPENDENCY).withType(JobParameter.Type.STRING) + .add(MODULES).withType(JobParameter.Type.STRING) + .add(REPORT_OUTPUT_TYPES).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportConfiguration.ReportOutputType.S3.name()) + .add(REPORT_FORMAT_TYPE).withType(JobParameter.Type.HIDDEN).withDefaultValue(ReportConfiguration.ReportFormatType.JSON.name()) + .build(); + + return new Job() + .withCategory(new JobCategory(JobType.REPORT, JobCategory.DEVOPS)) + .withName("Browser Release Stats Generator") + .withDescription("This report generates release statistics for the browser.") + .withParameters(params) + .withTag(INT) + .withTag(MS) + .withProductionStatus(Job.ProductionStatus.PROD_READY) + .withExpectedDuration(30) + .build(); + } + + @Override + protected void loadProjectSnapshot(boolean fsnOnly) throws TermServerScriptException { + prevDependency = getJobRun().getParamValue(PREV_DEPENDENCY); + if (!StringUtils.isEmpty(prevDependency)) { + LOGGER.info("Setting previous dependency archive to {}", prevDependency); + setDependencyArchive(prevDependency); + } + super.loadProjectSnapshot(fsnOnly); + } + + @Override + protected void loadCurrentPosition(boolean compareTwoSnapshots, boolean fsnOnly) throws TermServerScriptException { + thisDependency = getJobRun().getParamValue(THIS_DEPENDENCY); + if (!StringUtils.isEmpty(thisDependency)) { + LOGGER.info("Setting current dependency archive to {}", thisDependency); + setDependencyArchive(thisDependency); + } + super.loadCurrentPosition(compareTwoSnapshots, fsnOnly); + } + +} diff --git a/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/configuration/WebSecurityConfig.java b/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/configuration/WebSecurityConfig.java index 30f1b13640..03bd074a25 100644 --- a/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/configuration/WebSecurityConfig.java +++ b/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/configuration/WebSecurityConfig.java @@ -71,4 +71,8 @@ public String getOverrideUsername() { return overrideUsername; } + public String getOverrideRoles() { + return overrideRoles; + } + } diff --git a/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/rest/SchedulerController.java b/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/rest/SchedulerController.java index 931baf474e..4ac667a7b1 100644 --- a/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/rest/SchedulerController.java +++ b/reporting-service/src/main/java/org/ihtsdo/authoring/scheduler/api/rest/SchedulerController.java @@ -53,27 +53,33 @@ public class SchedulerController { private static final Logger LOGGER = LoggerFactory.getLogger(SchedulerController.class); - private static final String X_AUTH_TOK = "X-AUTH-token"; - private static final String X_AUTH_USER = "X-AUTH-username"; + private static final String X_AUTH_TOKEN = "X-AUTH-token"; + private static final String X_AUTH_USERNAME = "X-AUTH-username"; + private static final String X_AUTH_ROLES = "X-AUTH-roles"; + private static final String ROLE_SNOWSTORM_SUPPORT = "ROLE_snowstorm-support"; private static class AuthData { public final String authToken; public final String userName; + public final String roles; - public AuthData(String authToken, String userName) { + public AuthData(String authToken, String userName, String roles) { this.authToken = authToken; this.userName = userName; + this.roles = roles; } } private AuthData getAuthData(HttpServletRequest request) throws BusinessServiceException { - String authToken = request.getHeader(X_AUTH_TOK); - String userName = request.getHeader(X_AUTH_USER); + String authToken = request.getHeader(X_AUTH_TOKEN); + String userName = request.getHeader(X_AUTH_USERNAME); + String roles = request.getHeader(X_AUTH_ROLES); if (StringUtils.isEmpty(authToken) || StringUtils.isEmpty(userName)) { //Are local override values available? authToken = config.getOverrideToken(); userName = config.getOverrideUsername(); + roles = config.getOverrideRoles(); if (StringUtils.isEmpty(authToken) || StringUtils.isEmpty(userName)) { throw new BusinessServiceException("Failed to recover authentication details from HTTP headers"); @@ -82,7 +88,7 @@ private AuthData getAuthData(HttpServletRequest request) throws BusinessServiceE } } - return new AuthData(authToken, userName); + return new AuthData(authToken, userName, roles); } @Operation(summary="List Job Types") @@ -95,7 +101,7 @@ public List listJobTypes() { @Operation(summary="List job type categories") @ApiResponse(responseCode = "200", description = "OK") @GetMapping(value="/jobs/{typeName}") - public synchronized List listJobTypeCategories(@PathVariable final String typeName) throws BusinessServiceException { + public synchronized List listJobTypeCategories(HttpServletRequest request, @PathVariable final String typeName) throws BusinessServiceException { //Do we need to refresh the cache? if (new Date().getTime() - lastCacheUpdate.getTime() > CACHE_TIMEOUT) { jobCache.clear(); @@ -103,16 +109,19 @@ public synchronized List listJobTypeCategories(@PathVariable final } //Do we have the data cached? - if (jobCache.containsKey(typeName)) { - return jobCache.get(typeName); + if (!jobCache.containsKey(typeName)) { + LOGGER.info("Populating cache of known jobs for type: {}. Refresh scheduled for 30mins.", typeName); + List jobCategories = scheduleService.listJobTypeCategories(typeName).stream() + .filter(jc -> !jc.getJobs().isEmpty()) + .map(this::reverseParameterOptions) + .toList(); + jobCache.put(typeName, jobCategories); } - LOGGER.info("Populating cache of known jobs for type: {}. Refresh scheduled for 30mins.", typeName); - List jobCategories = scheduleService.listJobTypeCategories(typeName).stream() - .filter(jc -> !jc.getJobs().isEmpty()) - .map(this::reverseParameterOptions) - .toList(); - jobCache.put(typeName, jobCategories); - return jobCategories; + + if (!getAuthData(request).roles.contains(ROLE_SNOWSTORM_SUPPORT)) { + return jobCache.get(typeName).stream().filter(jobCategory -> !JobCategory.DEVOPS.equals(jobCategory.getName())).toList(); + } + return jobCache.get(typeName); } private JobCategory reverseParameterOptions(JobCategory jobCategory) { @@ -128,7 +137,6 @@ private JobCategory reverseParameterOptions(JobCategory jobCategory) { jobParameter.setValues(reversedDefaultValues); } } - } } return jobCategory;