diff --git a/lib/ramble/ramble/cmd/simplify.py b/lib/ramble/ramble/cmd/simplify.py new file mode 100644 index 000000000..55096be7f --- /dev/null +++ b/lib/ramble/ramble/cmd/simplify.py @@ -0,0 +1,986 @@ +# Copyright 2022-2026 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import ast +import difflib +import fnmatch +import inspect +import os +import re +import sys + +import ramble.keywords +import ramble.repository +import ramble.util.colors as color +from ramble.expander import ( + format_spec_regex, + supported_list_function_pointers, + supported_modules, + supported_scalar_function_pointers, + supported_scalar_function_with_self_arg_pointers, +) +from ramble.util.logger import logger + +description = "find and simplify unused or unreachable sections in definitions" +section = "developer" +level = "long" + + +def extract_referenced_names(template_str): + if not isinstance(template_str, str): + return set() + + referenced = set() + brace_contents = [] + stack = [] + escaped = False + for i, char in enumerate(template_str): + if char == "{": + if not escaped: + stack.append(i) + elif char == "}": + if not escaped and stack: + start = stack.pop() + brace_contents.append(template_str[start + 1 : i]) + elif char == "\n": + stack = [] + + if char == "\\": + escaped = True + else: + escaped = False + + for content in brace_contents: + match = format_spec_regex.search(content) + if match: + content = match.group("kw") + + # Replace '::' with '.' to make namespace references valid Python attributes + ast_content = content.replace("::", ".") + try: + tree = ast.parse(ast_content, mode="eval") + + def get_refs(node, parent=None): + r = set() + if isinstance(node, ast.Name): + if isinstance(parent, ast.Attribute) and node is parent.value: + pass + else: + r.add(node.id) + elif isinstance(node, ast.Attribute): + if isinstance(parent, ast.Attribute) and node is parent.value: + pass + else: + r.add(node.attr) + r.update(get_refs(node.value, parent=node)) + else: + for child in ast.iter_child_nodes(node): + r.update(get_refs(child, parent=node)) + return r + + referenced.update(get_refs(tree.body)) + except SyntaxError: + # Fallback to regex-based extraction if syntax is not valid Python + words = re.findall(r"[a-zA-Z0-9_:-]+", content) + for word in words: + if "::" in word: + referenced.add(word.split("::")[-1]) + else: + referenced.add(word) + return referenced + + +def find_class_file(parent_cls, obj_path=None): + module = sys.modules.get(parent_cls.__module__) + if module and hasattr(module, "__file__") and module.__file__: + return module.__file__ + + if not obj_path: + return None + + parts = parent_cls.__module__.split(".") + if len(parts) >= 4 and parts[0] == "ramble": + obj_abbrev = parts[1] + obj_name = parts[3] + for o_type in ramble.repository.ObjectTypes: + if o_type == ramble.repository.ObjectTypes.base_classes: + continue + abbrev = ramble.repository.type_definitions[o_type]["abbrev"] + if abbrev == obj_abbrev: + try: + path_inst = ramble.repository.paths[o_type] + return path_inst.filename_for_object_name(obj_name) + except Exception: + pass + return None + + +def find_template_file(cls, src_path_config, obj_path=None): + if os.path.isabs(src_path_config): + if os.path.isfile(src_path_config): + return src_path_config + return None + + # Get MRO to find where the class/parents are defined + for parent_cls in inspect.getmro(cls): + p_file_path = find_class_file(parent_cls, obj_path) + if p_file_path and (parent_cls.__module__.startswith("ramble") or "ramble" in p_file_path): + candidate = os.path.join(os.path.dirname(p_file_path), src_path_config) + if os.path.isfile(candidate): + return candidate + return None + + +def get_arg_value(arg_node): + if isinstance(arg_node, ast.Constant): + return arg_node.value + if hasattr(ast, "Str") and isinstance(arg_node, ast.Str): + return arg_node.s + return None + + +def get_node_end_lineno(node, file_lines): + if hasattr(node, "end_lineno") and node.end_lineno is not None: + return node.end_lineno + + # Fallback for Python < 3.8 which lacks end_lineno + start_idx = node.lineno - 1 + if start_idx >= len(file_lines): + return node.lineno + + open_parens = 0 + has_parens = False + in_single_quote = False + in_double_quote = False + in_triple_single = False + in_triple_double = False + + curr_line_idx = start_idx + while curr_line_idx < len(file_lines): + line = file_lines[curr_line_idx] + i = 0 + while i < len(line): + char = line[i] + # Handle comments: if we see '#' outside a string, ignore rest of line + if char == "#" and not ( + in_single_quote or in_double_quote or in_triple_single or in_triple_double + ): + break + + # Triple quotes + if not (in_single_quote or in_double_quote): + if line[i : i + 3] == "'''": + in_triple_single = not in_triple_single + i += 3 + continue + elif line[i : i + 3] == '"""': + in_triple_double = not in_triple_double + i += 3 + continue + + # Single/double quotes + if not (in_triple_single or in_triple_double): + if char == "'" and (i == 0 or line[i - 1] != "\\"): + in_single_quote = not in_single_quote + elif char == '"' and (i == 0 or line[i - 1] != "\\"): + in_double_quote = not in_double_quote + + # Parentheses counting + if not (in_single_quote or in_double_quote or in_triple_single or in_triple_double): + if char in "([{": + open_parens += 1 + has_parens = True + elif char in ")]}": + open_parens -= 1 + if has_parens and open_parens <= 0: + return curr_line_idx + 1 + i += 1 + + if not has_parens and curr_line_idx == start_idx: + # If no parens on the first line, assume single line statement + return node.lineno + curr_line_idx += 1 + + return len(file_lines) + + +def locate_directive_lines( + file_path, unused_vars, unused_inputs, unused_execs, unused_compilers, broken_vars +): + try: + with open(file_path, encoding="utf-8") as f: + content = f.read() + tree = ast.parse(content) + except (OSError, SyntaxError) as e: + logger.warn(f"Could not parse {file_path}: {e}") + return [] + + file_lines = content.splitlines() + + # Find the first class definition in the file + class_node = None + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + class_node = node + break + + if not class_node: + return [] + + ranges_to_remove = [] + + for node in class_node.body: + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + call = node.value + if isinstance(call.func, ast.Name): + func_name = call.func.id + if func_name in ("workload_variable", "variable"): + if call.args: + val = get_arg_value(call.args[0]) + if val in unused_vars or val in broken_vars: + ranges_to_remove.append( + (node.lineno, get_node_end_lineno(node, file_lines)) + ) + elif func_name == "input_file" and unused_inputs: + if call.args: + val = get_arg_value(call.args[0]) + if val in unused_inputs: + ranges_to_remove.append( + (node.lineno, get_node_end_lineno(node, file_lines)) + ) + elif func_name in ("executable", "formatted_executable") and unused_execs: + if call.args: + val = get_arg_value(call.args[0]) + if val in unused_execs: + ranges_to_remove.append( + (node.lineno, get_node_end_lineno(node, file_lines)) + ) + elif func_name == "define_compiler" and unused_compilers: + if call.args: + val = get_arg_value(call.args[0]) + if val in unused_compilers: + ranges_to_remove.append( + (node.lineno, get_node_end_lineno(node, file_lines)) + ) + + return ranges_to_remove + + +def apply_simplifications(file_path, ranges): + with open(file_path, encoding="utf-8") as f: + original_content = f.read() + + original_lines = original_content.splitlines(keepends=True) + new_lines = list(original_lines) + + # Sort ranges in descending order of start line + for start, end in sorted(ranges, reverse=True): + del new_lines[start - 1 : end] + + new_content = "".join(new_lines) + return original_lines, new_lines, new_content + + +def is_valid_reference( + var_name, + all_defined_variables, + defined_inputs, + defined_software_specs, + source_code, + fom_captures=None, +): + if var_name in all_defined_variables or var_name in defined_inputs: + return True + if var_name.endswith("_path") and var_name[:-5] in defined_software_specs: + return True + if var_name in ramble.keywords.default_keys: + return True + kw_inst = ramble.keywords.Keywords() + if any(pat.match(var_name) for pat in kw_inst.reserved_patterns): + return True + if fom_captures and var_name in fom_captures: + return True + + # Math functions and constants are always valid references + _allowed_math_names = getattr(is_valid_reference, "_allowed_math_names", None) + if _allowed_math_names is None: + _allowed_math_names = set( + list(supported_scalar_function_pointers.keys()) + + list(supported_list_function_pointers.keys()) + + list(supported_scalar_function_with_self_arg_pointers.keys()) + + list(supported_modules.keys()) + + ["True", "False", "None"] + ) + is_valid_reference._allowed_math_names = _allowed_math_names + + if var_name in _allowed_math_names: + return True + + # Strip quotes or match as literal token in source code (fallback for python references) + if re.search(r"\b" + re.escape(var_name) + r"\b", source_code): + return True + return False + + +def analyze_object(name, obj_type): + obj_path = ramble.repository.paths[obj_type] + cls = obj_path.get_obj_class(name) + + target_file_path = obj_path.filename_for_object_name(name) + + # Collect source codes from the class and all its parent classes in ramble/ + source_codes = [] + for parent_cls in inspect.getmro(cls): + p_file_path = find_class_file(parent_cls, obj_path) + if p_file_path and (parent_cls.__module__.startswith("ramble") or "ramble" in p_file_path): + try: + with open(p_file_path, encoding="utf-8") as f: + source_codes.append(f.read()) + except OSError as e: + logger.warn(f"Could not read source file {p_file_path}: {e}") + source_code = "\n".join(source_codes) + + # Clean the source code (strip comments/strings) for fallback scan + cleaned_code = re.sub(r"#.*", "", source_code) + cleaned_code = re.sub(r'""".*?"""', "", cleaned_code, flags=re.DOTALL) + cleaned_code = re.sub(r"'''.*?'''", "", cleaned_code, flags=re.DOTALL) + # Strip variable reference templates in braces (e.g. {var}) + cleaned_code = re.sub(r"\{[a-zA-Z0-9_:-]+\}", "", cleaned_code) + + # Parse AST of the target file + try: + with open(target_file_path, encoding="utf-8") as f: + file_content = f.read() + tree = ast.parse(file_content) + except Exception: + tree = None + + # Collect all workloads and workload groups (including inherited ones) + all_defined_workloads = set() + if hasattr(cls, "workloads") and cls.workloads: + for app_workloads in cls.workloads.values(): + for wl_name in app_workloads: + all_defined_workloads.add(wl_name) + + all_defined_workload_groups = set() + if hasattr(cls, "workload_groups") and cls.workload_groups: + for group_name in cls.workload_groups: + all_defined_workload_groups.add(group_name) + + # Collect compilers and software specs (including inherited ones) + all_defined_compilers = set() + if hasattr(cls, "compilers") and cls.compilers: + for compiler_name in cls.compilers: + all_defined_compilers.add(compiler_name) + + all_referenced_compilers = set() + if hasattr(cls, "software_specs") and cls.software_specs: + for specs_list in cls.software_specs.values(): + for spec_obj in specs_list: + if hasattr(spec_obj, "compiler") and spec_obj.compiler: + all_referenced_compilers.add(spec_obj.compiler) + + # 1. Collect all defined inputs (only for applications) + defined_inputs = set() + if hasattr(cls, "inputs") and cls.inputs: + for inputs_dict in cls.inputs.values(): + for input_name in inputs_dict: + defined_inputs.add(input_name) + + # 2. Collect all defined executables (only for applications) + defined_executables = set() + if hasattr(cls, "executables") and cls.executables: + for execs_dict in cls.executables.values(): + for exec_name in execs_dict: + defined_executables.add(exec_name) + + # 3. Collect all defined variables + defined_variables = set() + variable_instances = {} + + # Workload variables (for applications) + if hasattr(cls, "workloads") and cls.workloads: + for app_workloads in cls.workloads.values(): + for wl_obj in app_workloads.values(): + for var_list in wl_obj.variables.values(): + for var in var_list: + defined_variables.add(var.name) + variable_instances.setdefault(var.name, []).append(var) + + # Object variables (for modifiers, base classes, etc.) + if hasattr(cls, "object_variables") and cls.object_variables: + for var_list in cls.object_variables.values(): + for var in var_list: + defined_variables.add(var.name) + variable_instances.setdefault(var.name, []).append(var) + + # Collect all software spec names (including inherited ones), package names, + # and required packages + defined_software_specs = set() + if hasattr(cls, "software_specs") and cls.software_specs: + for specs_list in cls.software_specs.values(): + for spec_obj in specs_list: + defined_software_specs.add(spec_obj.name) + # If spec name has braces/template parts, add prefix up to the first brace/hyphen + # E.g. 'orca-{version}' -> add 'orca' + prefix = spec_obj.name.split("{")[0].rstrip("-") + if prefix: + defined_software_specs.add(prefix) + + # Extract package name from pkg_spec (e.g. 'orca@5.0.4' -> 'orca') + if spec_obj.pkg_spec: + pkg_match = re.match(r"\s*([\w-]+)", spec_obj.pkg_spec) + if pkg_match: + defined_software_specs.add(pkg_match.group(1)) + + if hasattr(cls, "required_packages") and cls.required_packages: + for pkgname in cls.required_packages: + defined_software_specs.add(pkgname) + + # 4. Gather all templates/strings to extract referenced names and check + # broken template references + all_referenced_names = set() + broken_template_refs = set() + + # Extract from executables templates + if hasattr(cls, "executables") and cls.executables: + for execs_dict in cls.executables.values(): + for exec_obj in execs_dict.values(): + templates = ( + exec_obj.template + if isinstance(exec_obj.template, list) + else [exec_obj.template] + ) + for t in templates: + refs = extract_referenced_names(t) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + # Extract from variables default values + if hasattr(cls, "workloads") and cls.workloads: + for app_workloads in cls.workloads.values(): + for wl_obj in app_workloads.values(): + for var_list in wl_obj.variables.values(): + for var in var_list: + refs = extract_referenced_names(str(var.default)) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + if hasattr(cls, "object_variables") and cls.object_variables: + for var_list in cls.object_variables.values(): + for var in var_list: + refs = extract_referenced_names(str(var.default)) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, defined_variables, defined_inputs, defined_software_specs, cleaned_code + ): + broken_template_refs.add(r) + + # Extract from object environment variables + if hasattr(cls, "object_environment_variables") and cls.object_environment_variables: + for env_vars_list in cls.object_environment_variables.values(): + for env_var in env_vars_list: + refs = set() + if env_var.name: + refs.update(extract_referenced_names(env_var.name)) + if env_var.value: + refs.update(extract_referenced_names(str(env_var.value))) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + # Extract from workload group environment variables + if hasattr(cls, "workload_group_env_vars") and cls.workload_group_env_vars: + for env_vars_list in cls.workload_group_env_vars.values(): + for env_var in env_vars_list: + refs = set() + if env_var.name: + refs.update(extract_referenced_names(env_var.name)) + if env_var.value: + refs.update(extract_referenced_names(str(env_var.value))) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + # Extract from workload environment variables + if hasattr(cls, "workloads") and cls.workloads: + for app_workloads in cls.workloads.values(): + for wl_obj in app_workloads.values(): + if hasattr(wl_obj, "environment_variables") and wl_obj.environment_variables: + for env_vars_list in wl_obj.environment_variables.values(): + for env_var in env_vars_list: + refs = set() + if env_var.name: + refs.update(extract_referenced_names(env_var.name)) + if env_var.value: + refs.update(extract_referenced_names(str(env_var.value))) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + # Extract from inputs url/description + if hasattr(cls, "inputs") and cls.inputs: + for inputs_dict in cls.inputs.values(): + for input_obj in inputs_dict.values(): + url = None + if hasattr(input_obj, "url"): + url = input_obj.url + elif isinstance(input_obj, dict) and "url" in input_obj: + url = input_obj["url"] + if url: + refs = extract_referenced_names(url) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + # Extract from figures of merit log_file + if hasattr(cls, "figures_of_merit") and cls.figures_of_merit: + for contexts_dict in cls.figures_of_merit.values(): + for foms_dict in contexts_dict.values(): + for fom_val in foms_dict.values(): + fom_captures: set[str] = set() + if isinstance(fom_val, dict) and "fom_regex" in fom_val: + try: + fom_captures.update(re.compile(fom_val["fom_regex"]).groupindex.keys()) + except re.error as e: + logger.warn( + "Invalid regex for figure of merit: " + f"{fom_val['fom_regex']}. Error: {e}" + ) + if isinstance(fom_val, dict): + if "log_file" in fom_val: + refs = extract_referenced_names(fom_val["log_file"]) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + fom_captures, + ): + broken_template_refs.add(r) + + # Extract from success criteria files/formulas + if hasattr(cls, "success_criteria") and cls.success_criteria: + for criteria_dict in cls.success_criteria.values(): + if isinstance(criteria_dict, dict): + if "file" in criteria_dict: + refs = extract_referenced_names(criteria_dict["file"]) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + if "formula" in criteria_dict: + refs = extract_referenced_names(criteria_dict["formula"]) + all_referenced_names.update(refs) + for r in refs: + if criteria_dict.get("mode") == "fom_comparison" and r == "value": + continue + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + + # Extract from registered templates + if hasattr(cls, "templates") and cls.templates: + for templates_dict in cls.templates.values(): + for tpl_config in templates_dict.values(): + src_path_config = tpl_config.get("src_path") + if src_path_config: + tpl_file = find_template_file(cls, src_path_config, obj_path) + if tpl_file: + try: + with open(tpl_file, encoding="utf-8") as f_tpl: + tpl_content = f_tpl.read() + refs = extract_referenced_names(tpl_content) + all_referenced_names.update(refs) + for r in refs: + if not is_valid_reference( + r, + defined_variables, + defined_inputs, + defined_software_specs, + cleaned_code, + ): + broken_template_refs.add(r) + except Exception: + pass + + # 5. Extract workload relationships (executables and inputs used directly in workloads) + used_executables = set() + used_inputs = set() + if hasattr(cls, "workloads") and cls.workloads: + for app_workloads in cls.workloads.values(): + for wl_obj in app_workloads.values(): + if hasattr(wl_obj, "executables"): + used_executables.update(wl_obj.executables) + if hasattr(wl_obj, "inputs"): + used_inputs.update(wl_obj.inputs) + + # Also extract inputs referenced in other inputs/url or variables + for name in all_referenced_names: + if name in defined_inputs: + used_inputs.add(name) + + unused_variables = [] + for var in sorted(defined_variables): + if var in all_referenced_names: + continue + occurrences = len(re.findall(rf"\b{re.escape(var)}\b", cleaned_code)) + defs = len( + re.findall( + rf'(?:workload_variable|variable)\s*\(\s*[\'"]{re.escape(var)}[\'"]', cleaned_code + ) + ) + if occurrences <= defs: + v_instances = variable_instances.get(var, []) + if v_instances and all(getattr(v, "when", None) for v in v_instances): + continue + unused_variables.append(var) + + # For each defined input, check if it's used + unused_inputs = [] + for inp in sorted(defined_inputs): + if inp in used_inputs: + continue + occurrences = len(re.findall(rf"\b{re.escape(inp)}\b", cleaned_code)) + defs = len(re.findall(rf'input_file\s*\(\s*[\'"]{re.escape(inp)}[\'"]', cleaned_code)) + if occurrences <= defs: + unused_inputs.append(inp) + + # For each defined executable, check if it's used + unused_executables = [] + for exe in sorted(defined_executables): + if exe in used_executables: + continue + occurrences = len(re.findall(rf"\b{re.escape(exe)}\b", cleaned_code)) + defs = ( + len(re.findall(rf'executable\s*\(\s*[\'"]{re.escape(exe)}[\'"]', cleaned_code)) + + len(re.findall(rf'edit_file\s*\(\s*[\'"]{re.escape(exe)}[\'"]', cleaned_code)) + + len(re.findall(rf'patch_file\s*\(\s*[\'"]{re.escape(exe)}[\'"]', cleaned_code)) + + len( + re.findall( + rf'formatted_executable\s*\(\s*[\'"]{re.escape(exe)}[\'"]', cleaned_code + ) + ) + ) + if occurrences <= defs: + unused_executables.append(exe) + + # Statically parse class body to find unused compilers, broken variables, and workload groups + unused_compilers = [] + broken_vars = [] + broken_groups = [] + + if tree: + class_node = None + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + class_node = node + break + + if class_node: + for node in class_node.body: + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + call = node.value + if isinstance(call.func, ast.Name): + func_name = call.func.id + + # Compiler check + if func_name == "define_compiler" and call.args: + compiler_name = get_arg_value(call.args[0]) + if compiler_name and compiler_name not in all_referenced_compilers: + occurrences = len( + re.findall(rf"\b{re.escape(compiler_name)}\b", cleaned_code) + ) + pattern = ( + rf"define_compiler\s*\(\s*[\'\"]" + rf"{re.escape(compiler_name)}[\'\"]" + ) + defs = len(re.findall(pattern, cleaned_code)) + if occurrences <= defs: + unused_compilers.append(compiler_name) + + # Variable broken workload/group reference check + elif func_name in ("workload_variable", "variable"): + is_broken = False + var_name = get_arg_value(call.args[0]) if call.args else None + for kw in call.keywords: + if kw.arg == "workload": + wl_val = get_arg_value(kw.value) + if ( + wl_val + and wl_val != "*" + and not any( + fnmatch.fnmatch(w, wl_val) + for w in all_defined_workloads + ) + ): + is_broken = True + elif kw.arg == "workload_group": + wg_val = get_arg_value(kw.value) + if ( + wg_val + and wg_val != "*" + and not any( + fnmatch.fnmatch(g, wg_val) + for g in all_defined_workload_groups + ) + ): + is_broken = True + elif kw.arg == "workloads": + if isinstance(kw.value, ast.List): + for elt in kw.value.elts: + wl_val = get_arg_value(elt) + if ( + wl_val + and wl_val != "*" + and not any( + fnmatch.fnmatch(w, wl_val) + for w in all_defined_workloads + ) + ): + is_broken = True + if is_broken and var_name: + broken_vars.append(var_name) + + # Workload group broken workload reference check + elif func_name == "workload_group": + group_name = get_arg_value(call.args[0]) if call.args else None + for kw in call.keywords: + if kw.arg == "workloads": + if isinstance(kw.value, ast.List): + for elt in kw.value.elts: + wl_val = get_arg_value(elt) + if ( + wl_val + and wl_val != "*" + and not any( + fnmatch.fnmatch(w, wl_val) + for w in all_defined_workloads + ) + ): + broken_groups.append(f"{group_name} -> {wl_val}") + + return ( + unused_variables, + unused_inputs, + unused_executables, + unused_compilers, + broken_vars, + broken_groups, + sorted(broken_template_refs), + ) + + +def setup_parser(subparser): + subparser.add_argument( + "-t", + "--type", + default="applications", + choices=ramble.repository.OBJECT_NAMES, + help="Object type to check (default: applications)", + ) + subparser.add_argument( + "-r", + "--repo", + default=None, + help="Only check objects defined within the specified repository namespace", + ) + subparser.add_argument( + "-d", + "--diff", + action="store_true", + help="show diff of proposed simplifications", + ) + subparser.add_argument( + "-a", + "--apply", + action="store_true", + help="apply proposed simplifications to definition files", + ) + subparser.add_argument( + "names", + nargs="*", + default=None, + help="Names of specific objects to check. If none, check all objects.", + ) + + +def simplify(parser, args): + obj_type = ramble.repository.ObjectTypes[args.type] + obj_path = ramble.repository.paths[obj_type] + + if args.repo: + try: + repo = obj_path.get_repo(args.repo) + names_in_repo = set(repo.all_object_names()) + if args.names: + names = [n for n in args.names if n in names_in_repo] + else: + names = list(names_in_repo) + except Exception: + logger.error(f"Repository namespace '{args.repo}' is not configured.") + return 1 + else: + if args.names: + names = args.names + else: + names = obj_path.all_object_names() + + total_unused_vars = 0 + total_unused_inputs = 0 + total_unused_execs = 0 + total_unused_compilers = 0 + total_broken_vars = 0 + + for name in sorted(names): + try: + res = analyze_object(name, obj_type) + ( + unused_vars, + unused_inputs, + unused_execs, + unused_compilers, + broken_vars, + broken_groups, + broken_templates, + ) = res + + if ( + unused_vars + or unused_inputs + or unused_execs + or unused_compilers + or broken_vars + or broken_groups + or broken_templates + ): + file_path = obj_path.filename_for_object_name(name) + color.cprint(f"@c{{=== {args.type.capitalize().rstrip('s')}: {name} ===}}") + if unused_vars: + print(f" Unused Variables: {unused_vars}") + total_unused_vars += len(unused_vars) + if unused_inputs: + print(f" Unused Inputs: {unused_inputs}") + total_unused_inputs += len(unused_inputs) + if unused_execs: + print(f" Unused Executables: {unused_execs}") + total_unused_execs += len(unused_execs) + if unused_compilers: + print(f" Unused Compilers: {unused_compilers}") + total_unused_compilers += len(unused_compilers) + if broken_vars: + print(f" Variables with Broken Workload/Group Refs: {broken_vars}") + total_broken_vars += len(broken_vars) + if broken_groups: + print(f" Workload Groups with Broken Workload Refs: {broken_groups}") + if broken_templates: + print(f" Broken Variable References in Templates: {broken_templates}") + + # Locate line ranges to remove (including compilers and broken variables) + ranges = locate_directive_lines( + file_path, + unused_vars, + unused_inputs, + unused_execs, + unused_compilers, + broken_vars, + ) + + if ranges: + original_lines, new_lines, new_content = apply_simplifications( + file_path, ranges + ) + + if args.diff: + print(" Proposed Changes:") + diff = difflib.unified_diff( + original_lines, + new_lines, + fromfile=file_path, + tofile=file_path + ".simplified", + ) + for line in diff: + sys.stdout.write(" " + line) + + if args.apply: + with open(file_path, "w", encoding="utf-8") as f: + f.write(new_content) + print(f" Successfully simplified {file_path}") + + print() + except Exception as e: + logger.warn(f"Error analyzing {name}: {e}") + + color.cprint( + f"@g{{Summary: Found {total_unused_vars} unused variables, " + f"{total_unused_inputs} unused inputs, " + f"{total_unused_execs} unused executables, " + f"{total_unused_compilers} unused compilers, " + f"and {total_broken_vars} variables with broken references.}}" + ) + + return 0 diff --git a/lib/ramble/ramble/test/cmd/simplify.py b/lib/ramble/ramble/test/cmd/simplify.py new file mode 100644 index 000000000..4933a1f76 --- /dev/null +++ b/lib/ramble/ramble/test/cmd/simplify.py @@ -0,0 +1,811 @@ +# Copyright 2022-2026 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import sys + +import pytest + +import ramble.error +import ramble.repository +from ramble.main import RambleCommand + +pytestmark = pytest.mark.usefixtures("config") + +simplify_cmd = RambleCommand("simplify") + + +def test_simplify_runs(): + with pytest.raises(SystemExit): + simplify_cmd("-h") + + +def test_simplify_applications(mock_applications): + out = simplify_cmd("-t", "applications", "basic") + assert "=== Application: basic ===" in out + assert "Unused Variables: [" in out + assert "my_var" in out + assert "my_base_var" in out + + +def test_simplify_diff_and_apply(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testns\n") + + app_dir = os.path.join(repo_path, "applications", "testapp") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + original_code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class Testapp(ExecutableApplication): + name = "testapp" + executable('foo', 'bar', use_mpi=False) + workload('test_wl', executable='foo') + workload_variable('my_var', default='1.0', workload='test_wl') +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(original_code) + + # Need to clean the cached instance for applications in paths mapping + obj_type = ramble.repository.ObjectTypes.applications + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + # Overlay the test repo on top of applications repositories + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + # 1. Test running simplify without flags (shows unused variables) + out = simplify_cmd("-t", "applications", "testapp") + assert "Unused Variables: ['my_var']" in out + + # 2. Test running simplify with --diff flag + out_diff = simplify_cmd("-t", "applications", "-d", "testapp") + assert "Proposed Changes:" in out_diff + assert "- workload_variable('my_var', default='1.0', workload='test_wl')" in out_diff + + # Verify file is still unmodified + with open(app_file, encoding="utf-8") as f: + assert "workload_variable('my_var'" in f.read() + + # 3. Test running simplify with --apply flag + out_apply = simplify_cmd("-t", "applications", "-a", "testapp") + assert "Successfully simplified" in out_apply + + # Verify file is now simplified + with open(app_file, encoding="utf-8") as f: + content = f.read() + assert "workload_variable('my_var'" not in content + assert 'name = "testapp"' in content + + # Clean sys.modules cache and recreate repo to force reloading from disk + sys_module_name = "ramble.app.testns.testapp" + sys.modules.pop(sys_module_name, None) + parent = sys.modules.get("ramble.app.testns") + if parent and "testapp" in parent.__dict__: + del parent.__dict__["testapp"] + + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo_new = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo_new, object_type=obj_type): + # 4. Test running simplify again, should find 0 unused elements + out_clean = simplify_cmd("-t", "applications", "testapp") + assert "Found 0 unused variables" in out_clean + + +def test_simplify_compilers_and_broken_refs(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo2")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testns2\n") + + app_dir = os.path.join(repo_path, "applications", "testapp2") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + original_code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class Testapp2(ExecutableApplication): + name = "testapp2" + define_compiler('gcc14', pkg_spec='gcc@14.1.0') + define_compiler('clang15', pkg_spec='llvm@15.0.0') + + software_spec('my_pkg', pkg_spec='zlib@1.2.11', compiler='gcc14') + + executable('foo', 'bar {var1}', use_mpi=False) + workload('test_wl', executable='foo') + + workload_variable('var1', default='1.0', workload='test_wl') + workload_variable('var2', default='2.0', workload='nonexistent_wl') + workload_group('group1', workloads=['nonexistent_wl2']) +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(original_code) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + out = simplify_cmd("-t", "applications", "testapp2") + # Assert no unused variables (since var1 is used in 'bar {var1}') + assert "Unused Variables" not in out or "var1" not in out + # Assert unused compilers detected + assert "Unused Compilers: ['clang15']" in out + # Assert broken variables detected + assert "Variables with Broken Workload/Group Refs: ['var2']" in out + # Assert broken workload groups detected + assert "Workload Groups with Broken Workload Refs: ['group1 -> nonexistent_wl2']" in out + + # Test applying simplifications + out_apply = simplify_cmd("-t", "applications", "-a", "testapp2") + assert "Successfully simplified" in out_apply + + # Verify clang15 and var2 were deleted from file + with open(app_file, encoding="utf-8") as f: + content = f.read() + assert "define_compiler('clang15'" not in content + assert "workload_variable('var2'" not in content + # gcc14 and var1 should be untouched (since var1 is used and + # define_compiler('gcc14') is used!) + assert "define_compiler('gcc14'" in content + assert "workload_variable('var1'" in content + # workload group is untouched (not auto-deleted) + assert "workload_group('group1'" in content + + +def test_simplify_broken_template_references(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo3")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testns3\n") + + app_dir = os.path.join(repo_path, "applications", "testapp3") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + original_code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class Testapp3(ExecutableApplication): + name = "testapp3" + required_package('wrf') + software_spec('orca-{application::orca::version}', pkg_spec='orca@5.0.4') + input_file('my_input', url='https://host.com/file.tar.gz', description='my input file') + executable( + 'foo', + 'bar {my_var} {my_input} {nonexistent_var_typo} {wrf_path} ' + '{orca_path} {my_var - 1} {my_var -1} {my_var-1}', + use_mpi=False, + ) + workload('test_wl', executable='foo') + workload_variable('my_var', default='1.0', workload='test_wl') + workload_variable('my_formatted_var', default='{{{my_var}/2}:0.0f}', workload='test_wl') +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(original_code) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + out = simplify_cmd("-t", "applications", "testapp3") + # Assert broken template reference is detected + assert "Broken Variable References in Templates: ['nonexistent_var_typo']" in out + # Assert format specifier parts are NOT extracted/reported as broken references + assert ":0" not in out + assert "0f" not in out + # Assert valid variables, inputs, and spec paths are NOT reported as broken references + assert "my_var" not in out or "Broken Variable References" not in out.split("my_var")[0] + assert "my_input" not in out + assert "wrf_path" not in out + assert "orca_path" not in out + + +def test_simplify_environment_variable_references(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo_env_vars")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testns_env_vars\n") + + app_dir = os.path.join(repo_path, "applications", "testapp_env") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + original_code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class TestappEnv(ExecutableApplication): + name = "testapp_env" + executable('foo', 'bar', use_mpi=False) + workload('test_wl', executable='foo') + workload_variable( + 'var_in_env_val', + environment_variable_name='MY_ENV_VAR', + default='1.0', + workload='test_wl' + ) + workload_variable( + 'var_in_env_name', + environment_variable_name='MY_ENV_{var_in_env_name_ref}', + default='2.0', + workload='test_wl' + ) + workload_variable( + 'var_in_env_name_ref', + default='suffix', + workload='test_wl' + ) +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(original_code) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + out = simplify_cmd("-t", "applications", "testapp_env") + # Assert none of the variables are reported as unused + assert "Unused Variables" not in out + assert "Found 0 unused variables" in out + + +def test_simplify_repo_filter(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo4")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testrepo\n") + + app_dir = os.path.join(repo_path, "applications", "testapp4") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + original_code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class Testapp4(ExecutableApplication): + name = "testapp4" + executable('foo', 'bar', use_mpi=False) + workload('test_wl', executable='foo') + workload_variable('my_var', default='1.0', workload='test_wl') +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(original_code) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + # Scan with repository filter set to 'testrepo' + out = simplify_cmd("-t", "applications", "-r", "testrepo") + assert "=== Application: testapp4 ===" in out + assert "Unused Variables: ['my_var']" in out + + # Scan with nonexistent repo namespace should return failure + with pytest.raises(ramble.error.RambleCommandError): + simplify_cmd("-t", "applications", "-r", "nonexistent") + + +def test_simplify_wildcard_workloads(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo5")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testns5\n") + + app_dir = os.path.join(repo_path, "applications", "testapp5") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + original_code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class Testapp5(ExecutableApplication): + name = "testapp5" + executable('foo', 'bar {dict_delim} {nonexistent_var}', use_mpi=False) + workload('motorbike_20m', executable='foo') + workload_variable('dict_delim', default='.', workloads=['motorbike*']) + workload_variable('nonexistent_var', default='.', workloads=['other*']) +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(original_code) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + out = simplify_cmd("-t", "applications", "testapp5") + # Assert nonexistent_var is detected as broken because 'other*' matches zero workloads + assert "Variables with Broken Workload/Group Refs: ['nonexistent_var']" in out + # dict_delim should NOT be detected as broken because 'motorbike*' matches 'motorbike_20m' + assert ( + "dict_delim" not in out + or "Variables with Broken Workload" not in out.split("dict_delim")[0] + ) + + +def test_get_node_end_lineno_fallback(): + import ast + + from ramble.cmd.simplify import get_node_end_lineno + + content = """executable( + 'foo', + 'bar', + use_mpi=False +) +""" + tree = ast.parse(content) + node = tree.body[0] + + # Force fallback by deleting end_lineno if it exists + if hasattr(node, "end_lineno"): + del node.end_lineno + + file_lines = content.splitlines() + end_line = get_node_end_lineno(node, file_lines) + assert end_line == 5 + + +def test_extract_referenced_names(): + from ramble.cmd.simplify import extract_referenced_names + + assert extract_referenced_names("foo {application::orca::version}") == {"version"} + assert extract_referenced_names("foo {simple_var}") == {"simple_var"} + assert extract_referenced_names(123) == set() + + +def test_find_template_file_direct(tmpdir): + from ramble.cmd.simplify import find_template_file + + class DummyClass: + __module__ = "ramble.app.dummy" + + # Create dummy module in sys.modules + import types + + dummy_module = types.ModuleType("ramble.app.dummy") + dummy_module.__file__ = os.path.join(str(tmpdir), "application.py") + sys.modules["ramble.app.dummy"] = dummy_module + + # Test absolute path + abs_path = os.path.join(str(tmpdir), "absolute_template.in") + with open(abs_path, "w", encoding="utf-8") as f: + f.write("test") + assert find_template_file(DummyClass, abs_path) == abs_path + assert find_template_file(DummyClass, "/nonexistent/abs/path") is None + + # Test relative path + rel_path = "relative_template.in" + full_rel_path = os.path.join(str(tmpdir), rel_path) + with open(full_rel_path, "w", encoding="utf-8") as f: + f.write("test") + assert find_template_file(DummyClass, rel_path) == full_rel_path + assert find_template_file(DummyClass, "nonexistent_rel.in") is None + + # Clean up sys.modules + sys.modules.pop("ramble.app.dummy", None) + + +def test_get_arg_value_direct(): + import ast + + from ramble.cmd.simplify import get_arg_value + + tree = ast.parse("foo(bar)") + stmt = tree.body[0] + assert isinstance(stmt, ast.Expr) + call = stmt.value + assert isinstance(call, ast.Call) + # call.args[0] is ast.Name (bar) + assert get_arg_value(call.args[0]) is None + + +def test_get_node_end_lineno_fallback_detailed(): + import ast + + from ramble.cmd.simplify import get_node_end_lineno + + # 1. Comment outside strings + content1 = """executable( + 'foo', # this is a comment + 'bar' + )""" + tree1 = ast.parse(content1) + node1 = tree1.body[0] + if hasattr(node1, "end_lineno"): + del node1.end_lineno + assert get_node_end_lineno(node1, content1.splitlines()) == 4 + + # 2. Escaped quotes and different quote types + content2 = """executable('foo', "bar \\' baz", 'qux')""" + tree2 = ast.parse(content2) + node2 = tree2.body[0] + if hasattr(node2, "end_lineno"): + del node2.end_lineno + assert get_node_end_lineno(node2, content2.splitlines()) == 1 + + # 3. Triple quotes + content3 = """executable( + 'foo', + \"""triple + double + quote\""", + '''triple + single + quote''', + 'bar' + )""" + tree3 = ast.parse(content3) + node3 = tree3.body[0] + if hasattr(node3, "end_lineno"): + del node3.end_lineno + assert get_node_end_lineno(node3, content3.splitlines()) == 10 + + # 4. No parens first line fallback + content4 = "x = 1" + tree4 = ast.parse(content4) + node4 = tree4.body[0] + if hasattr(node4, "end_lineno"): + del node4.end_lineno + assert get_node_end_lineno(node4, content4.splitlines()) == 1 + + +def test_locate_directive_lines_errors(tmpdir, monkeypatch): + import ramble.util.logger + from ramble.cmd.simplify import locate_directive_lines + + # Mock logger.warn to verify it was called + warn_calls = [] + monkeypatch.setattr(ramble.util.logger.logger, "warn", lambda msg: warn_calls.append(msg)) + + # Test file that does not exist (OSError) + res = locate_directive_lines("/nonexistent/file/path", set(), set(), set(), set(), []) + assert res == [] + assert any("Could not parse" in c for c in warn_calls) + + # Test file with syntax error + bad_file = os.path.join(str(tmpdir), "bad_syntax.py") + with open(bad_file, "w", encoding="utf-8") as f: + f.write("class BadClass:\n def foo(:\n") + + warn_calls.clear() + res = locate_directive_lines(bad_file, set(), set(), set(), set(), []) + assert res == [] + assert any("Could not parse" in c for c in warn_calls) + + +def test_locate_directive_lines_no_class(tmpdir): + from ramble.cmd.simplify import locate_directive_lines + + empty_file = os.path.join(str(tmpdir), "empty.py") + with open(empty_file, "w", encoding="utf-8") as f: + f.write("# just comment, no class\n") + + assert locate_directive_lines(empty_file, set(), set(), set(), set(), []) == [] + + +def test_simplify_comprehensive(tmpdir, mutable_config): + repo_path = str(tmpdir.join("ramble_repo_comprehensive")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testcomp\n") + + app_dir = os.path.join(repo_path, "applications", "compapp") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + + # Write registered template + tpl_file = os.path.join(app_dir, "my_template.in") + with open(tpl_file, "w", encoding="utf-8") as f: + f.write( + "template referencing {var1} and {broken_tpl_ref} and {application::orca::version}\n" + ) + + code = """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * + +class Compapp(ExecutableApplication): + name = "compapp" + + # Compilers + define_compiler('gcc14', pkg_spec='gcc@14') + define_compiler('unused_compiler', pkg_spec='gcc@13') + + # Software specs + software_spec('orca-{version}', pkg_spec='orca@5.0.4') + software_spec('my_pkg', pkg_spec='zlib', compiler='gcc14') + + # Inputs + input_file('my_input', url='https://host.com/file-{var1}.tar.gz', description='my input') + input_file('unused_input', url='https://host.com/unused-{broken_input_ref}.tar.gz', description='unused input') + + # Executables + executable('foo', 'bar {var1} {my_input} {application_version}', use_mpi=False) + executable('unused_exec', 'baz', use_mpi=False) + + # Workloads + workload('test_wl', executable='foo') + + # Workload groups + workload_group('group1', workloads=['test_wl']) + + # Variables + workload_variable('var1', default='1.0', workload='test_wl') + workload_variable('var2', default='{var1}', workload='nonexistent_wl') + workload_variable('version', default='5.0', workload='test_wl') + + # Non-constant variable definition + local_name = 'some_local_name' + workload_variable(local_name, default='2.0', workload='test_wl') + + # Figure of Merit + figure_of_merit('my_fom', fom_regex=r'Result: (?P\\d+)', group_name='fom_val', log_file='{var1}.log') + figure_of_merit('fom_with_refs', fom_regex=r'Result: (?P\\d+)', group_name='fom_val', log_file='{fom_val} {broken_fom_ref}.log') + + # Success Criteria + success_criteria('my_crit', 'fom_comparison', file='{var1}.log', formula='{var1} > 0') + success_criteria('fom_value_crit', 'fom_comparison', file='{var1}.log', fom_name='my_fom', formula='{value} > 0') + success_criteria('broken_crit', 'fom_comparison', file='{broken_crit_ref}.log', formula='{broken_formula_ref}') + + # Templates + register_template('tpl1', src_path='my_template.in', dest_path='my_template.out') + + # Fallback check for python reference + def some_method(self): + x = 'referenced_in_python_code' + + workload_variable('referenced_in_python_code', default='val', workload='test_wl') +""" + with open(app_file, "w", encoding="utf-8") as f: + f.write(code) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + # 1. Run simplify without flags, verify all outputs + out = simplify_cmd("-t", "applications", "compapp") + + # Verify unused compilers, inputs, executables + assert "Unused Compilers: ['unused_compiler']" in out + assert "Unused Inputs: ['unused_input']" in out + assert "Unused Executables: ['unused_exec']" in out + assert "Unused Variables" not in out or "referenced_in_python_code" not in out + + # Verify variables with broken workload group references + assert "Variables with Broken Workload/Group Refs: ['var2']" in out + + # Verify broken variable references in templates + assert "broken_crit_ref" in out + assert "broken_fom_ref" in out + assert "broken_formula_ref" in out + assert "broken_tpl_ref" in out + assert "broken_input_ref" in out + + # 2. Run simplify --apply, verify file changes + out_apply = simplify_cmd("-t", "applications", "-a", "compapp") + assert "Successfully simplified" in out_apply + + with open(app_file, encoding="utf-8") as f: + content = f.read() + # Verify unused/broken entities are deleted + assert "define_compiler('unused_compiler'" not in content + assert "input_file('unused_input'" not in content + assert "executable('unused_exec'" not in content + assert "workload_variable('var2'" not in content + + # Verify valid entities remain + assert "define_compiler('gcc14'" in content + assert "input_file('my_input'" in content + assert "executable('foo'" in content + assert "workload_variable('var1'" in content + assert "workload_variable('referenced_in_python_code'" in content + + +def test_simplify_modifier(tmpdir, mutable_config): + repo_path = str(tmpdir.join("ramble_mod_repo")) + os.makedirs(os.path.join(repo_path, "modifiers")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: testmodns\n") + + mod_dir = os.path.join(repo_path, "modifiers", "testmod") + os.makedirs(mod_dir) + mod_file = os.path.join(mod_dir, "modifier.py") + + code = """# Copyright 2022-2026 The Ramble Authors +from ramble.modkit import * + +class Testmod(BasicModifier): + name = "testmod" + + # Modifiers use 'variable' instead of 'workload_variable' + # We define a variable that is unused, and one that is used in a template + variable('used_var', default='1.0', description='used variable') + variable('unused_var', default='{broken_mod_ref}', description='unused variable') + + def some_method(self): + self.used_var +""" + with open(mod_file, "w", encoding="utf-8") as f: + f.write(code) + + obj_type = ramble.repository.ObjectTypes.modifiers + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + out = simplify_cmd("-t", "modifiers", "testmod") + assert "=== Modifier: testmod ===" in out + assert "Unused Variables: ['unused_var']" in out + assert "Broken Variable References in Templates: ['broken_mod_ref']" in out + + # Test apply + out_apply = simplify_cmd("-t", "modifiers", "-a", "testmod") + assert "Successfully simplified" in out_apply + + with open(mod_file, encoding="utf-8") as f: + content = f.read() + assert "variable('unused_var'" not in content + assert "variable('used_var'" in content + + +def test_simplify_repo_filter_with_names_and_all_names(tmpdir, mutable_config): + repo_path = str(tmpdir.join("test_repo_names")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: repotests\n") + + app_dir = os.path.join(repo_path, "applications", "app1") + os.makedirs(app_dir) + with open(os.path.join(app_dir, "application.py"), "w", encoding="utf-8") as f: + f.write( + """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * +class App1(ExecutableApplication): + name = "app1" +""" + ) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + # Scan with repository filter and specific names (covers line 720) + out1 = simplify_cmd("-t", "applications", "-r", "repotests", "app1") + assert "Found 0 unused variables" in out1 + + # Scan without specifying any names (covers line 730) + out2 = simplify_cmd("-t", "applications") + assert "Summary:" in out2 + + +def test_simplify_analysis_error(tmpdir, mutable_config, monkeypatch): + import ramble.cmd.simplify + import ramble.util.logger + + warn_calls = [] + monkeypatch.setattr(ramble.util.logger.logger, "warn", lambda msg: warn_calls.append(msg)) + + def mock_analyze(name, obj_type): + raise RuntimeError("Simulated analysis error") + + monkeypatch.setattr(ramble.cmd.simplify, "analyze_object", mock_analyze) + + # Let's run simplify on some application. Since it's mocked to + # raise error, it should log it as warning + repo_path = str(tmpdir.join("test_repo_err")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: errns\n") + + app_dir = os.path.join(repo_path, "applications", "errapp") + os.makedirs(app_dir) + with open(os.path.join(app_dir, "application.py"), "w", encoding="utf-8") as f: + f.write( + """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * +class Errapp(ExecutableApplication): + name = "errapp" +""" + ) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + simplify_cmd("-t", "applications", "errapp") + assert any("Error analyzing errapp: Simulated analysis error" in c for c in warn_calls) + + +def test_simplify_open_source_oserror(tmpdir, mutable_config, monkeypatch): + import builtins + + import ramble.util.logger + + repo_path = str(tmpdir.join("ramble_repo_oserror")) + os.makedirs(os.path.join(repo_path, "applications")) + with open(os.path.join(repo_path, "repo.yaml"), "w", encoding="utf-8") as f: + f.write("repo:\n namespace: oserrns\n") + + app_dir = os.path.join(repo_path, "applications", "oserrapp") + os.makedirs(app_dir) + app_file = os.path.join(app_dir, "application.py") + with open(app_file, "w", encoding="utf-8") as f: + f.write( + """# Copyright 2022-2026 The Ramble Authors +from ramble.appkit import * +class Oserrapp(ExecutableApplication): + name = "oserrapp" +""" + ) + + obj_type = ramble.repository.ObjectTypes.applications + try: + ramble.repository.paths[obj_type]._instance = None + except Exception: + pass + + test_repo = ramble.repository.Repo(repo_path, object_type=obj_type) + + original_open = builtins.open + + warn_calls = [] + monkeypatch.setattr(ramble.util.logger.logger, "warn", lambda msg: warn_calls.append(msg)) + + def mock_open(file, *args, **kwargs): + if isinstance(file, (str, bytes, os.PathLike)) and os.path.realpath( + file + ) == os.path.realpath(app_file): + raise OSError("Simulated read failure") + return original_open(file, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", mock_open) + + with ramble.repository.use_repositories(test_repo, object_type=obj_type): + simplify_cmd("-t", "applications", "oserrapp") + assert any("Could not read source file" in c for c in warn_calls) diff --git a/share/ramble/ramble-completion.bash b/share/ramble/ramble-completion.bash index 4ef1dfb77..9dc0c02dd 100755 --- a/share/ramble/ramble-completion.bash +++ b/share/ramble/ramble-completion.bash @@ -268,7 +268,7 @@ _ramble() { then RAMBLE_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --disable-passthrough -N --disable-logger -A --aggregate-warnings -S --suppress-warnings -P --disable-progress-bar --timestamp --pdb -w --workspace -D --workspace-dir -W --no-workspace --use-workspace-repo --resolve-variables-in-subprocesses -k --insecure -l --enable-locks -L --disable-locks -m --mock --overwrite-inventories --mock-applications --mock-modifiers --mock-package-managers --mock-workflow-managers --mock-systems --mock-platforms --mock-base-classes --mock-base-applications --mock-base-modifiers --mock-base-package-managers --mock-base-workflow-managers --mock-base-systems --mock-base-platforms --mock-utilities --mock-base-utilities -p --profile --sorted-profile --lines --profile-restrictions -v --verbose --stacktrace -V --version" else - RAMBLE_COMPREPLY="attributes clean commands config create data debug deployment docs edit filter-groups help info license list mirror on python repo results software-definitions style unit-test workspace" + RAMBLE_COMPREPLY="attributes clean commands config create data debug deployment docs edit filter-groups help info license list mirror on python repo results simplify software-definitions style unit-test workspace" fi } @@ -631,6 +631,15 @@ _ramble_results_report() { RAMBLE_COMPREPLY="-h --help --workspace --strong-scaling --weak-scaling --multi-line --compare --foms --pandas-where -n --normalize --logx --logy --simplify-names --split-by -f --file" } +_ramble_simplify() { + if $list_options + then + RAMBLE_COMPREPLY="-h --help -t --type -r --repo -d --diff -a --apply" + else + RAMBLE_COMPREPLY="" + fi +} + _ramble_software_definitions() { RAMBLE_COMPREPLY="-h --help -s --summary -c --conflicts -e --error-on-conflict" } diff --git a/var/ramble/repos/builtin/applications/maxtext/application.py b/var/ramble/repos/builtin/applications/maxtext/application.py index 713b40c18..c6d18a0a1 100644 --- a/var/ramble/repos/builtin/applications/maxtext/application.py +++ b/var/ramble/repos/builtin/applications/maxtext/application.py @@ -114,12 +114,14 @@ class Maxtext(ExecutableApplication): default="{maxtext_path}:{maxtext_path}", description="Container mount for maxtext root", workloads=all_workloads, + when="workflow_manager=slurm-pyxis", ) workload_variable( "container_mounts", default="{maxtext_mount}", description="All container mounts in a ramble variable", workloads=all_workloads, + when="workflow_manager=slurm-pyxis", ) log_str = os.path.join("{experiment_run_dir}", "metrics.out")