lint: replace black/isort/flake8 with ruff

.devcontainer/devcontainer.json

@@ -24,7 +24,7 @@
 		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
 		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
 		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.ruffPath": "/usr/local/py-utils/bin/ruff",
 		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
 		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
 		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",

.github/CONTRIBUTING.md

@@ -195,7 +195,7 @@ All Python code must adhere to the style guide used by capa:
 
   1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from
   2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with
-  3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with
+  3. [ruff](https://docs.astral.sh/ruff/) (with line length 120 and imports ordered by line length), and formatted with
   4. [black](https://github.com/psf/black) (with line width 120), and formatted with
   5. [dos2unix](https://linux.die.net/man/1/dos2unix)
 

.github/ruff.toml

@@ -1,18 +1,5 @@
-# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
-# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
-# McCabe complexity (`C901`) by default.
-lint.select = ["E", "F"]
-
-# Allow autofix for all enabled rules (when `--fix`) is provided.
-lint.fixable = ["ALL"]
-lint.unfixable = []
-
-# E402 module level import not at top of file
-# E722 do not use bare 'except'
-# E501 line too long
-lint.ignore = ["E402", "E722", "E501"]
-
 line-length = 120
+preview = true # Required to enable pre-release copyright header checks (CPY001)
 
 exclude = [
     # Exclude a variety of commonly ignored directories.
@@ -41,3 +28,71 @@ exclude = [
     "*_pb2.py", 
     "*_pb2.pyi"
 ]
+
+lint.select = [
+    "E",   # pycodestyle (base style rules)
+    "F",   # Pyflakes (logical/syntax errors)
+    "I",   # isort (import sorting)
+    "B",   # flake8-bugbear (common bugs/design problems)
+    "C4",  # flake8-comprehensions (simplify list/dict comprehensions)
+    "ISC", # flake8-implicit-str-concat (detect accidental multi-line string issues)
+    "T20", # flake8-print (prevent leftover print/pprint statements)
+    "SIM", # flake8-simplify (code simplification upgrades)
+    "CPY", # flake8-copyright (header requirement enforcement)
+    "G",   # flake8-logging-format (logging statement validation)
+    "TD",  # flake8-todos (TODO formatting requirements)
+    "PTH", # flake8-use-pathlib (migration from os.path to Pathlib)
+    "UP"   # pyupgrade (modern Python syntax upgrades)
+]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+lint.fixable = ["ALL"]
+lint.unfixable = []
+
+# Map existing flake8 ignores to maintain strict parity
+lint.ignore = [
+    # Legacy flake8 ignores
+    "E402", # Module level import not at top of file
+    "E722", # Do not use bare except
+    "E501", # Line too long
+    "E203", # Whitespace before ':'
+    "E701", # Multiple statements on one line
+    "B010", # Do not call setattr with a constant attribute value
+    "SIM102", # Use a single if statement instead of nested if statements
+    "SIM114", # Combine if branches using logical or operator
+
+    # Newly surfaced Ruff strictness ignores
+    "B905", # zip() without an explicit strict= parameter
+    "UP032", # Use f-string instead of format call
+    "UP031", # Use format specifiers instead of percent format
+    "SIM300", # Yoda condition detected (constant before variable)
+    "SIM108", # Use ternary operator instead of if-else block
+    "ISC003", # Explicitly concatenated string should be implicitly concatenated
+    "UP035", # Deprecated typing alias usage
+    "UP006", # Use type instead of Type for type annotation
+    "SIM115", # Use a context manager for opening files
+    "SIM118", # Use key not in dict instead of key not in dict.keys()
+    "UP024", # Replace aliased errors with OSError
+    "UP045", # Use X | None for optional type annotations
+    "SIM103", # Return negated condition directly
+    "UP007", # Use X | Y for union type annotations
+    "B904", # Raise exceptions within except clause using raise from
+    "UP028", # Replace yield over for loop with yield from
+    "C409", # Unnecessary list comprehension passed to tuple()
+    "E226", # Missing whitespace around arithmetic operator
+]
+
+[lint.per-file-ignores]
+# T201 print found schemas for scripts and entrypoints
+"scripts/*" = ["T201"]
+"capa/main.py" = ["T201"]
+"capa/features/extractors/binja/find_binja_api.py" = ["T201"]
+"tests/conftest.py" = ["I001"] # Suppress import sorting to preserve explicit legacy fixture loading order
+"*_pb2.py" = ["ALL"] # Completely disable all formatting for auto-generated protocol buffer files
+
+[lint.flake8-copyright]
+notice-rgx = "Copyright \\d{4} Google LLC"
+min-file-size = 1
+
+[lint.isort]
+length-sort = true

.github/workflows/black-format.yml

@@ -35,8 +35,10 @@ jobs:
           pip install -r requirements.txt
           pip install -e .[dev,scripts]
 
-      - name: Run isort
-        run: pre-commit run isort --all-files
+      - name: Run ruff/continue
+        # ruff returns non-zero error code after formatting, which is what we expect
+        continue-on-error: true
+        run: pre-commit run ruff --all-files
 
       - name: Run black/continue
         # black returns non-zero error code after formatting, which is what we expect
@@ -58,5 +60,5 @@ jobs:
           git config user.name "${GITHUB_ACTOR}"
           git config user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com"
           git add -A
-          git commit -m "style: auto-format with black and isort"
+          git commit -m "style: auto-format with ruff and black"
           git push

.github/workflows/tests.yml

@@ -52,12 +52,8 @@ jobs:
         pip install -e .[dev,scripts]
     - name: Lint with ruff
       run: pre-commit run ruff
-    - name: Lint with isort
-      run: pre-commit run isort --show-diff-on-failure
     - name: Lint with black
       run: pre-commit run black --show-diff-on-failure
-    - name: Lint with flake8
-      run: pre-commit run flake8 --hook-stage manual
     - name: Check types with mypy
       run: pre-commit run mypy --hook-stage manual
     - name: Check imports against dependencies

.justfile

@@ -1,25 +1,17 @@
-@isort:
-    pre-commit run isort --show-diff-on-failure --all-files
-
 @black:
     pre-commit run black --show-diff-on-failure --all-files
 
 @ruff:
     pre-commit run ruff --all-files
 
-@flake8:
-    pre-commit run flake8 --hook-stage manual --all-files
-
 @mypy:
     pre-commit run mypy --hook-stage manual --all-files
 
 @deptry:
     pre-commit run deptry --hook-stage manual --all-files
 
 @lint:
-    -just isort
     -just black
     -just ruff
-    -just flake8
     -just mypy
     -just deptry

.pre-commit-config.yaml

@@ -9,38 +9,16 @@
 # run all linters liks:
 #
 #    ❯ pre-commit run --all-files
-#    isort....................................................................Passed
 #    black....................................................................Passed
 #    ruff.....................................................................Passed
-#    flake8...................................................................Passed
 #    mypy.....................................................................Passed
 #
 # run a single linter like:
 #
-#    ❯ pre-commit run --all-files isort
-#    isort....................................................................Passed
+#    ❯ pre-commit run --all-files ruff
+#    ruff.....................................................................Passed
 
 repos:
--   repo: local
-    hooks:
-    -   id: isort
-        name: isort
-        stages: [pre-commit, pre-push, manual]
-        language: system
-        entry: isort
-        args: 
-        -   "--length-sort"
-        -   "--profile"
-        -   "black"
-        -   "--line-length=120"
-        -   "--skip-glob"
-        -   "*_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        -   "web/rules/scripts/"
-        always_run: true
-        pass_filenames: false
 
 -   repo: local
     hooks:
@@ -75,27 +53,10 @@ repos:
         -   "scripts/"
         -   "tests/"
         -   "web/rules/scripts/"
+        exclude: '.*_pb2\.py$'
         always_run: true
         pass_filenames: false
 
--   repo: local
-    hooks:
-    -   id: flake8
-        name: flake8
-        stages: [pre-push, manual]
-        language: system
-        entry: flake8
-        args: 
-        -   "--config"
-        -   ".github/flake8.ini"
-        -   "--extend-exclude"
-        -   "capa/render/proto/capa_pb2.py,capa/features/extractors/binexport2/binexport2_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        -   "web/rules/scripts/"
-        always_run: true
-        pass_filenames: false
 
 -   repo: local
     hooks:

CHANGELOG.md

@@ -17,6 +17,7 @@
 ### capa Explorer IDA Pro plugin
 
 ### Development
+- replace isort/flake8 with ruff @mike-hunhoff #2992
 
 ### Raw diffs
 - [capa v9.4.0...master](https://github.com/mandiant/capa/compare/v9.4.0...master)

capa/capabilities/common.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

capa/capabilities/dynamic.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,6 @@
 import collections
 from dataclasses import dataclass
 
-import capa.perf
 import capa.engine
 import capa.helpers
 import capa.features.freeze as frz

capa/capabilities/static.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

capa/engine.py

@@ -216,7 +216,7 @@ def evaluate(self, features: FeatureSet, short_circuit=True):
             # because we've overridden `__bool__` above.
             #
             # we can't use `if child is True` because the instance is not True.
-            success = sum([1 for child in results if bool(child) is True]) >= self.count
+            success = sum(1 for child in results if bool(child) is True) >= self.count
             return Result(success, self, results)
 
 

capa/features/extractors/binexport2/arch/intel/insn.py

@@ -15,7 +15,6 @@
 import logging
 from typing import Iterator
 
-import capa.features.extractors.strings
 import capa.features.extractors.binexport2.helpers
 from capa.features.insn import MAX_STRUCTURE_SIZE, Number, Offset, OperandNumber, OperandOffset
 from capa.features.common import Feature, Characteristic
@@ -170,12 +169,12 @@ def is_security_cookie(
     basic_block_index: int = bbi.basic_block_index
     bb: BinExport2.BasicBlock = be2.basic_block[basic_block_index]
     if flow_graph.entry_basic_block_index == basic_block_index:
-        first_addr: int = min((idx.insn_address_by_index[ir.begin_index] for ir in bb.instruction_index))
+        first_addr: int = min(idx.insn_address_by_index[ir.begin_index] for ir in bb.instruction_index)
         if instruction_address < first_addr + SECURITY_COOKIE_BYTES_DELTA:
             return True
     # or insn falls at the end before return in a terminal basic block.
     if basic_block_index not in (e.source_basic_block_index for e in flow_graph.edge):
-        last_addr: int = max((idx.insn_address_by_index[ir.end_index - 1] for ir in bb.instruction_index))
+        last_addr: int = max(idx.insn_address_by_index[ir.end_index - 1] for ir in bb.instruction_index)
         if instruction_address > last_addr - SECURITY_COOKIE_BYTES_DELTA:
             return True
     return False

capa/features/extractors/binexport2/extractor.py

@@ -15,7 +15,6 @@
 import logging
 from typing import Iterator
 
-import capa.features.extractors.elf
 import capa.features.extractors.common
 import capa.features.extractors.binexport2.file
 import capa.features.extractors.binexport2.insn

capa/features/extractors/binexport2/file.py

@@ -19,7 +19,6 @@
 import pefile
 from elftools.elf.elffile import ELFFile
 
-import capa.features.common
 import capa.features.extractors.common
 import capa.features.extractors.pefile
 import capa.features.extractors.elffile

capa/features/extractors/binexport2/helpers.py

@@ -18,7 +18,6 @@
 from dataclasses import dataclass
 
 import capa.features.extractors.helpers
-import capa.features.extractors.binexport2.helpers
 from capa.features.common import ARCH_I386, ARCH_AMD64, ARCH_AARCH64
 from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
 

capa/features/extractors/binja/extractor.py

@@ -16,7 +16,6 @@
 
 import binaryninja as binja
 
-import capa.features.extractors.elf
 import capa.features.extractors.binja.file
 import capa.features.extractors.binja.insn
 import capa.features.extractors.binja.global_

capa/features/extractors/binja/file.py

@@ -16,9 +16,7 @@
 
 from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
 
-import capa.features.extractors.common
 import capa.features.extractors.helpers
-import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section, FunctionName
 from capa.features.common import (
     FORMAT_PE,

capa/features/extractors/dnfile/extractor.py

@@ -21,7 +21,6 @@
 import dnfile
 from dncil.cil.opcode import OpCodes
 
-import capa.features.extractors
 import capa.features.extractors.dotnetfile
 import capa.features.extractors.dnfile.file
 import capa.features.extractors.dnfile.insn