From a9bd4e2e19940279408702f2dad6b530261db7ec Mon Sep 17 00:00:00 2001 From: wejoncy Date: Wed, 4 Mar 2026 09:46:52 +0000 Subject: [PATCH 1/3] fix: compatibility with transformers >= 5 and support non-llama models in chat plugin 1. Catcher: add __getattr__ to proxy wrapped module attributes (e.g. attention_type for Qwen3), fixing AttributeError during input hijacking. 2. base.py: add _no_init_weights() fallback for transformers >= 5 where no_init_weights was removed. 3. inference.py: enable chat_loop_v3 for non-llama models with apply_chat_template support. --- qllm/modeling/base.py | 19 ++++++++++++++++++- qllm/plugin/chatcli/inference.py | 4 ++-- qllm/quantization/quant_frame_base.py | 8 ++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/qllm/modeling/base.py b/qllm/modeling/base.py index ba121de..0e1ef49 100644 --- a/qllm/modeling/base.py +++ b/qllm/modeling/base.py @@ -21,6 +21,23 @@ logger = utils.logger.get_logger() +@contextlib.contextmanager +def _no_init_weights(): + """Replace transformers.modeling_utils.no_init_weights removed in transformers >= 5.""" + import torch.nn.init as init + _orig = {} + skip = lambda *a, **kw: None + for name in list(vars(init)): + if name.endswith('_') and callable(getattr(init, name)): + _orig[name] = getattr(init, name) + setattr(init, name, skip) + try: + yield + finally: + for name, fn in _orig.items(): + setattr(init, name, fn) + + @contextlib.contextmanager def replace_default_dtype(dtype): old_dtype = torch.get_default_dtype() @@ -244,7 +261,7 @@ def from_quantized( model_name_or_path, trust_remote_code=trust_remote_code) torch_dtype = torch_dtype if torch_dtype is not None else auto_conf.torch_dtype init_contexts = [ - transformers.modeling_utils.no_init_weights(), + transformers.modeling_utils.no_init_weights() if hasattr(transformers.modeling_utils, 'no_init_weights') else _no_init_weights(), # no_init_weights(), replace_default_dtype(torch_dtype), # accelerate.init_empty_weights(include_buffers=False) diff --git a/qllm/plugin/chatcli/inference.py b/qllm/plugin/chatcli/inference.py index 59ff287..07b264b 100644 --- a/qllm/plugin/chatcli/inference.py +++ b/qllm/plugin/chatcli/inference.py @@ -31,8 +31,8 @@ def chat_loop( if _fastchat_available: return chat_loop_v2(model, tokenizer) model_type = str(type(model)).lower() - #if "llama" not in model_type and hasattr(tokenizer, 'apply_chat_template'): - # return chat_loop_v3(model, tokenizer) + if "llama" not in model_type and hasattr(tokenizer, 'apply_chat_template'): + return chat_loop_v3(model, tokenizer) assert "llama" in model_type, 'have you installed fschat? please run `pip install fschat` and try again.' assert generate_stream_func is not None or generate_func is not None, 'should set generate function.' diff --git a/qllm/quantization/quant_frame_base.py b/qllm/quantization/quant_frame_base.py index bdb93ed..5bdf1cd 100644 --- a/qllm/quantization/quant_frame_base.py +++ b/qllm/quantization/quant_frame_base.py @@ -81,6 +81,14 @@ def __init__(self, module): super().__init__() self.module = module + def __getattr__(self, name): + if name == 'module': + return super().__getattr__(name) + try: + return super().__getattr__(name) + except AttributeError: + return getattr(self.module, name) + def forward(self, inp, **kwargs): inps.append(inp.to(swap_device)) layer_input_args.update(kwargs) From aca43982b22a7e3e31f2409d899faa4095dd11d3 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Wed, 4 Mar 2026 09:50:17 +0000 Subject: [PATCH 2/3] fix: remove unused nonlocal declarations to fix F824 flake8 errors --- qllm/quantization/vptq/hessian_collector.py | 3 +-- qllm/quantization/vptq/qllm_hessian.py | 3 +-- qllm/quantization/vptq/quant_vptq.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/qllm/quantization/vptq/hessian_collector.py b/qllm/quantization/vptq/hessian_collector.py index 905a348..f88570b 100644 --- a/qllm/quantization/vptq/hessian_collector.py +++ b/qllm/quantization/vptq/hessian_collector.py @@ -98,7 +98,7 @@ def register_H_hook(module, device): ct = 0 def H_hook(module, x): - nonlocal H, mu, ct, n + nonlocal ct x = x[0].reshape(-1, n).to(torch.float64) mu.add_(x.sum(dim=0)) H.addmm_(x.T, x) @@ -107,7 +107,6 @@ def H_hook(module, x): hook = module.register_forward_pre_hook(H_hook) def done(): - nonlocal H, mu, ct, hook hook.remove() return H.cpu(), mu.cpu(), ct diff --git a/qllm/quantization/vptq/qllm_hessian.py b/qllm/quantization/vptq/qllm_hessian.py index 15e8be8..c43cfc1 100644 --- a/qllm/quantization/vptq/qllm_hessian.py +++ b/qllm/quantization/vptq/qllm_hessian.py @@ -73,7 +73,7 @@ def register_H_hook(module, device): ct = 0 def H_hook(module, x): - nonlocal H, mu, ct, n + nonlocal ct x = x[0].reshape(-1, n).to(torch.float64) mu.add_(x.sum(dim=0)) H.addmm_(x.T, x) @@ -82,7 +82,6 @@ def H_hook(module, x): hook = module.register_forward_pre_hook(H_hook) def done(): - nonlocal H, mu, ct, hook hook.remove() return H.cpu(), mu.cpu(), ct diff --git a/qllm/quantization/vptq/quant_vptq.py b/qllm/quantization/vptq/quant_vptq.py index e8a8113..e0e1bbb 100644 --- a/qllm/quantization/vptq/quant_vptq.py +++ b/qllm/quantization/vptq/quant_vptq.py @@ -38,7 +38,6 @@ def forward(self, *args, **kwargs): raise ValueError def get_func(name, out_fetures): def fake_forward(hidden_state, *args, **kwargs): - nonlocal level_map if len(level_map) == 0: hidden_state *= 0 input_shape = hidden_state.shape From 7d1d8a82a45d0fa3c12df97a8132553e028035ad Mon Sep 17 00:00:00 2001 From: wejoncy Date: Wed, 4 Mar 2026 09:56:25 +0000 Subject: [PATCH 3/3] chore: add pre-commit hook and lint script (flake8, matching CI) --- .pre-commit-config.yaml | 6 ++++++ scripts/lint.sh | 12 ++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100755 scripts/lint.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..5b80f6c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.2 + hooks: + - id: flake8 + args: [--count, --select, "E9,F63,F7,F82", --show-source, --statistics] diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100755 index 0000000..1958216 --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +echo "=== Running flake8 (syntax errors & undefined names) ===" +flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + +echo "=== Running flake8 (warnings) ===" +flake8 ./qllm/modeling/q_layers --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +flake8 ./qllm/quantization --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +flake8 ./qllm/utils --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + +echo "All lint checks passed!"