From 85f20470f470c9214884cc8b6f3802d25110f8d8 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 3 Jul 2026 15:30:27 +0300 Subject: [PATCH 01/82] test update to lumi latest container --- BitsAndBytes/run-bnb-quantization-lumi.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/BitsAndBytes/run-bnb-quantization-lumi.sh b/BitsAndBytes/run-bnb-quantization-lumi.sh index 8492c78..8b8aafb 100644 --- a/BitsAndBytes/run-bnb-quantization-lumi.sh +++ b/BitsAndBytes/run-bnb-quantization-lumi.sh @@ -6,17 +6,20 @@ #SBATCH --gpus-per-node=1 #SBATCH --mem=32G #SBATCH --time=00:30:00 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err + # Load the module module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 +module use /appl/local/laifs/modules +module load lumi-aif-singularity-bindings + +# export path to used container image +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache +export HF_HOME=/scratch/project_462001302/mmahnoor/llm-quantization-scripts/BitsAndBytes/hf-cache mkdir -p $HF_HOME +export SINGULARITYENV_HF_HOME=$HF_HOME -srun python3 bnb-quantization.py +srun singularity exec -B /scratch/${SLURM_JOB_ACCOUNT} --cleanenv "$SIF" bash -lc 'python3 bnb-quantization.py' From 18db59ab0e509eb95c1cc1c54658781b4493ac6a Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:05:04 +0300 Subject: [PATCH 02/82] added use_cache= False for better results --- BitsAndBytes/bnb-quantization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/BitsAndBytes/bnb-quantization.py b/BitsAndBytes/bnb-quantization.py index d182d84..24ba45c 100644 --- a/BitsAndBytes/bnb-quantization.py +++ b/BitsAndBytes/bnb-quantization.py @@ -27,6 +27,7 @@ def benchmark(model, tokenizer, prompt): **inputs, max_new_tokens=50, do_sample=True, + use_cache=False, temperature=0.7, ) From 09f54d28c13f15194fd31ddf87213a69256140e8 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:07:01 +0300 Subject: [PATCH 03/82] updated script to new lumi aif container --- BitsAndBytes/run-bnb-quantization-lumi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BitsAndBytes/run-bnb-quantization-lumi.sh b/BitsAndBytes/run-bnb-quantization-lumi.sh index 8b8aafb..86a64b4 100644 --- a/BitsAndBytes/run-bnb-quantization-lumi.sh +++ b/BitsAndBytes/run-bnb-quantization-lumi.sh @@ -22,4 +22,4 @@ export HF_HOME=/scratch/project_462001302/mmahnoor/llm-quantization-scripts/Bits mkdir -p $HF_HOME export SINGULARITYENV_HF_HOME=$HF_HOME -srun singularity exec -B /scratch/${SLURM_JOB_ACCOUNT} --cleanenv "$SIF" bash -lc 'python3 bnb-quantization.py' +srun singularity exec "$SIF" bash -c 'python3 bnb-quantization.py' From e882be7b6512d8fca6b21a10d53ba2c6e541bd51 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:10:50 +0300 Subject: [PATCH 04/82] updated script to latest lumi aif container --- GPTQ/run-gptq-config-lumi.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/GPTQ/run-gptq-config-lumi.sh b/GPTQ/run-gptq-config-lumi.sh index 812ab59..d159e5b 100644 --- a/GPTQ/run-gptq-config-lumi.sh +++ b/GPTQ/run-gptq-config-lumi.sh @@ -9,17 +9,18 @@ #SBATCH --output=slurm-%j.out #SBATCH --error=slurm-%j.err -# Load the module +#Load the module module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 +module use /appl/local/laifs/modules +module load lumi-aif-singularity-bindings + +# export path to used container image +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache mkdir -p $HF_HOME -srun python3 gptq-config.py +srun singularity exec "$SIF" bash -c 'source optimum-env/bin/activate && python3 gptq-config.py' From 6989c4ba969ae639e716baea9282643644ac2253 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:21:38 +0300 Subject: [PATCH 05/82] Update README.md --- GPTQ/README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 4d2cad6..a73601c 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -8,7 +8,32 @@ Both examples currently use the small **OPT-125M** model for demonstration, but --- -## Installations +## LUMI + +To run gptq scripts on LUMI, you have to setup a Python environment using `optimum`, `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. + +1. Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. + +```bash +module purge +module use /appl/local/laifs/modules +module load lumi-aif-singularity-bindings + +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif + +singularity shell "$SIF" + +Singularity> python -m venv venv --system-site-packages +Singularity> source venv/bin/activate +(venv) Singularity> pip install optimum==1.27.0 +(venv) Singularity> pip install gptqmodel==4.0.0 --no-build-isolation --cache-dir ./.pip-cache +(venv) Singularity> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache + +``` + + +## Puhti & Mahti +### Installations The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples (torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. From 47435cc23b90cf47d9b0c2debdb840a1fc511246 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:21:49 +0300 Subject: [PATCH 06/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index a73601c..c1da542 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -12,7 +12,7 @@ Both examples currently use the small **OPT-125M** model for demonstration, but To run gptq scripts on LUMI, you have to setup a Python environment using `optimum`, `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. -1. Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. +Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. ```bash module purge From cf0f9b54fbd3baca754bb37f67ce1f88f2539234 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:22:17 +0300 Subject: [PATCH 07/82] minor changes --- GPTQ/run-gptq-config-lumi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/run-gptq-config-lumi.sh b/GPTQ/run-gptq-config-lumi.sh index d159e5b..e862660 100644 --- a/GPTQ/run-gptq-config-lumi.sh +++ b/GPTQ/run-gptq-config-lumi.sh @@ -23,4 +23,4 @@ export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-2026051 export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache mkdir -p $HF_HOME -srun singularity exec "$SIF" bash -c 'source optimum-env/bin/activate && python3 gptq-config.py' +srun singularity exec "$SIF" bash -c 'source venv/bin/activate && python3 gptq-config.py' From 52ffec1c9f4f8daaf817f227785d168d3156a646 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 6 Jul 2026 16:23:21 +0300 Subject: [PATCH 08/82] minor changes --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index c1da542..a7bfe11 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -30,7 +30,7 @@ Singularity> source venv/bin/activate (venv) Singularity> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache ``` - +The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. ## Puhti & Mahti ### Installations From 9de4ced5970aca490e2de7465f2378fcad959b9a Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 10:50:50 +0300 Subject: [PATCH 09/82] Create run-bnb-quantization-roihu.sh --- BitsAndBytes/run-bob-quantization-roihu.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 BitsAndBytes/run-bob-quantization-roihu.sh diff --git a/BitsAndBytes/run-bob-quantization-roihu.sh b/BitsAndBytes/run-bob-quantization-roihu.sh new file mode 100644 index 0000000..5facd27 --- /dev/null +++ b/BitsAndBytes/run-bob-quantization-roihu.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --argos=no +#SBATCH --account=project_xxxxxxx +#SBATCH --partition=gpumedium +#SBATCH --time=00:30:00 +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=288 +#SBATCH --gres=gpu:gh200:4 +#SBATCH --mem=240G + +module purge +module load python-pytorch/2.10 + +# We are putting the cache in the ramdisk, stored in memory. Alternatively store it to the project's scratch. +#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ +export HF_HOME=/dev/shm/$USER/hf-cache +export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ + +mkdir -p $HF_HOME + +srun python3 bnb-quantization.py From 6718b8c7c19c4c439c03eec945d3761d1c77a761 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 10:51:06 +0300 Subject: [PATCH 10/82] Rename run-bob-quantization-roihu.sh to run-bnb-quantization-roihu.sh --- ...un-bob-quantization-roihu.sh => run-bnb-quantization-roihu.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename BitsAndBytes/{run-bob-quantization-roihu.sh => run-bnb-quantization-roihu.sh} (100%) diff --git a/BitsAndBytes/run-bob-quantization-roihu.sh b/BitsAndBytes/run-bnb-quantization-roihu.sh similarity index 100% rename from BitsAndBytes/run-bob-quantization-roihu.sh rename to BitsAndBytes/run-bnb-quantization-roihu.sh From d3ac0cd9b98d70fc48b54e3c34b0ead7974b3251 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 11:17:22 +0300 Subject: [PATCH 11/82] Create run-gptq-config-roihu.sh --- GPTQ/run-gptq-config-roihu.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 GPTQ/run-gptq-config-roihu.sh diff --git a/GPTQ/run-gptq-config-roihu.sh b/GPTQ/run-gptq-config-roihu.sh new file mode 100644 index 0000000..c9e10f7 --- /dev/null +++ b/GPTQ/run-gptq-config-roihu.sh @@ -0,0 +1,25 @@ +#!/bin/bash +#SBATCH --argos=no +#SBATCH --account=project_2014553 +#SBATCH --partition=gpumedium +#SBATCH --time=00:30:00 +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=288 +#SBATCH --gres=gpu:gh200:4 +#SBATCH --mem=240G +module purge +module load python-pytorch/2.10 + +# Activate the virtual environment from your current directory or change to the appropriate path +source venv/bin/activate + +# We are putting the cache in the ramdisk, stored in +# memory. Alternatively store it to the project's scratch. +#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ +export HF_HOME=/dev/shm/$USER/hf-cache +export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +mkdir -p $HF_HOME + +srun python3 gptq-config-roihu.py + From a438e2dad319a1b3bbcb620747a667c67e523464 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 11:22:12 +0300 Subject: [PATCH 12/82] Update README.md --- GPTQ/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/GPTQ/README.md b/GPTQ/README.md index a7bfe11..684c627 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -32,6 +32,29 @@ Singularity> source venv/bin/activate ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. +-- +## Roihu + +The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples +(torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. + +### Load the module +```bash +module purge +module load python-pytorch/2.10 +``` +### Create and activate a virtual environment using system packages +```bash +python3 -m venv --system-site-packages venv +source venv/bin/activate +``` +### Install packages +```bash +pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +``` +This version of gptqmodel is compatible with python-pytorch/2.10. +--- + ## Puhti & Mahti ### Installations From 09770fcc4b28784c55f56a81364a7e1cada1d670 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 11:22:45 +0300 Subject: [PATCH 13/82] Update README.md --- GPTQ/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GPTQ/README.md b/GPTQ/README.md index 684c627..ca163a4 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -53,6 +53,8 @@ source venv/bin/activate pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache ``` This version of gptqmodel is compatible with python-pytorch/2.10. + + --- ## Puhti & Mahti From 69673eab6651915a8cde73d5f657a883b7d6ffb1 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 11:34:28 +0300 Subject: [PATCH 14/82] Create gptq-config-roihu.py --- GPTQ/gptq-config-roihu.py | 114 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 GPTQ/gptq-config-roihu.py diff --git a/GPTQ/gptq-config-roihu.py b/GPTQ/gptq-config-roihu.py new file mode 100644 index 0000000..3571197 --- /dev/null +++ b/GPTQ/gptq-config-roihu.py @@ -0,0 +1,114 @@ +import os +import time +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM +from gptqmodel import GPTQModel, QuantizeConfig +from datasets import load_dataset + +model_name = "facebook/opt-125m" +prompt = "The future of AI is" + + +# Measure model inference time and generate sample output for a given prompt +def benchmark(model, tokenizer, prompt, max_new_tokens=50): + inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + + # Warm-up run (to remove cold start effects) + with torch.no_grad(): + _ = model.generate(**inputs, max_new_tokens=5) + if torch.cuda.is_available(): + torch.cuda.synchronize() + + start = time.time() + with torch.no_grad(): + output_ids = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=True, + temperature=0.7, + ) + if torch.cuda.is_available(): + torch.cuda.synchronize() + end = time.time() + + elapsed_time = end - start + decoded_text = tokenizer.decode(output_ids[0], skip_special_tokens=True) + return decoded_text, elapsed_time + + +def get_folder_size(path): + total = 0 + for dirpath, _, filenames in os.walk(path): + for f in filenames: + total += os.path.getsize(os.path.join(dirpath, f)) + return total / (1024 * 1024) # MB + + +# --------------------------------------------------------------------------- +# 1. Load base model and tokenizer, run benchmark, save full model +# --------------------------------------------------------------------------- +model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto") +tokenizer = AutoTokenizer.from_pretrained(model_name) + +initial_output, initial_time = benchmark(model, tokenizer, prompt) + +save_dir_full = model_name.split("/")[-1] + "-full" +model.save_pretrained(save_dir_full, safe_serialization=True) +tokenizer.save_pretrained(save_dir_full) + +# Free memory before quantization +del model +if torch.cuda.is_available(): + torch.cuda.empty_cache() + +# --------------------------------------------------------------------------- +# 2. Prepare calibration dataset for GPTQ +# --------------------------------------------------------------------------- +calibration_dataset = load_dataset( + "allenai/c4", + data_files="en/c4-train.00001-of-01024.json.gz", + split="train", +).select(range(256))["text"] + +# --------------------------------------------------------------------------- +# 3. Quantize the model with gptqmodel (no optimum/transformers.GPTQConfig) +# --------------------------------------------------------------------------- +quantize_config = QuantizeConfig( + bits=4, + group_size=128, +) + +quant_model = GPTQModel.load(model_name, quantize_config) +quant_model.quantize(calibration_dataset, batch_size=2) + +save_dir_quant = model_name.split("/")[-1] + "-gptq-config" +quant_model.save(save_dir_quant) +tokenizer.save_pretrained(save_dir_quant) + +del quant_model +if torch.cuda.is_available(): + torch.cuda.empty_cache() + +# --------------------------------------------------------------------------- +# 4. Reload quantized model and benchmark +# --------------------------------------------------------------------------- +reloaded_quant_model = GPTQModel.load(save_dir_quant) +quant_tokenizer = AutoTokenizer.from_pretrained(save_dir_quant) + +quant_output, quant_time = benchmark(reloaded_quant_model, quant_tokenizer, prompt) + +# --------------------------------------------------------------------------- +# 5. Compare model sizes and print results +# --------------------------------------------------------------------------- +initial_size = get_folder_size(save_dir_full) +quant_size = get_folder_size(save_dir_quant) + +print("=== Full Model ===") +print(f" Output: {initial_output}") +print(f" Size: {initial_size:.2f} MB") +print(f" Inference time: {initial_time:.4f} s") + +print("\n=== Quantized Model ===") +print(f" Output: {quant_output}") +print(f" Size: {quant_size:.2f} MB") +print(f" Inference time: {quant_time:.4f} s") From 958a7912a08f6beb3c1bac9beeb7e59f9abdc9c0 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 11:53:29 +0300 Subject: [PATCH 15/82] Create run-gptq-modifier-roihu.sh --- GPTQ/run-gptq-modifier-roihu.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 GPTQ/run-gptq-modifier-roihu.sh diff --git a/GPTQ/run-gptq-modifier-roihu.sh b/GPTQ/run-gptq-modifier-roihu.sh new file mode 100644 index 0000000..25654ca --- /dev/null +++ b/GPTQ/run-gptq-modifier-roihu.sh @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH --argos=no +#SBATCH --account=project_2014553 +#SBATCH --partition=gpumedium +#SBATCH --time=00:30:00 +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=288 +#SBATCH --gres=gpu:gh200:4 +#SBATCH --mem=240G +module purge +module load python-pytorch/2.10 + +# Activate the virtual environment from your current directory or change to the appropriate path +source venv/bin/activate + +# We are putting the cache in the ramdisk, stored in +# memory. Alternatively store it to the project's scratch. +#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ +export HF_HOME=/dev/shm/$USER/hf-cache +export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +mkdir -p $HF_HOME + +srun python3 gptq-modifier.py From 6c83449e2376989f8f8163a84ba6a558c7f7e6d5 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 12:00:24 +0300 Subject: [PATCH 16/82] Update README.md --- GPTQ/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index ca163a4..a3add55 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -50,10 +50,26 @@ source venv/bin/activate ``` ### Install packages ```bash -pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +(venv)> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache ``` This version of gptqmodel is compatible with python-pytorch/2.10. +For the **gptq-modifier** example, you need to install the llmcompressor library. + +```bash +(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install wandb --cache-dir ./.pip-cache +``` + +Note: After installing packages, double-check that torch and torchvision are still coming from the module (/usr/local/lib64/python3.12/site-packages), not from your venv. Some packages you install (like gptqmodel or llmcompressor) can silently pull in their own copy of torch into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched torch/torchvision pair that breaks imports. + +To uninstall: +```bash +(venv)> pip uninstall torch -y +``` + + + --- From b77bf0d76c9e666add6ef7b417feecc5c4ba4b8e Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 12:02:03 +0300 Subject: [PATCH 17/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index a3add55..9aa3b93 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -61,7 +61,7 @@ For the **gptq-modifier** example, you need to install the llmcompressor library (venv)> pip install wandb --cache-dir ./.pip-cache ``` -Note: After installing packages, double-check that torch and torchvision are still coming from the module (/usr/local/lib64/python3.12/site-packages), not from your venv. Some packages you install (like gptqmodel or llmcompressor) can silently pull in their own copy of torch into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched torch/torchvision pair that breaks imports. +Note: After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. To uninstall: ```bash From 20f7910b4cdb3c963d1c308ee55cfbd685f5605f Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:24:27 +0300 Subject: [PATCH 18/82] Create run-awq-modifier-roihu.sh --- AWQ/run-awq-modifier-roihu.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 AWQ/run-awq-modifier-roihu.sh diff --git a/AWQ/run-awq-modifier-roihu.sh b/AWQ/run-awq-modifier-roihu.sh new file mode 100644 index 0000000..0922b9b --- /dev/null +++ b/AWQ/run-awq-modifier-roihu.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --argos=no +#SBATCH --account=project_xxxxxxx +#SBATCH --partition=gpumedium +#SBATCH --time=00:30:00 +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=1 +#SBATCH --cpus-per-task=288 +#SBATCH --gres=gpu:gh200:4 +#SBATCH --mem=240G + +module purge +module load python-pytorch/2.10 + +# Activate the virtual environment from your current directory or change to the appropriate path +source bbvenv/bin/activate + +# We are putting the cache in the ramdisk, stored in +# memory. Alternatively store it to the project's scratch. +#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ +export HF_HOME=/dev/shm/$USER/hf-cache +export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +mkdir -p $HF_HOME + +srun python3 awq-modifier.py + From 3f69b92ca43276581c312e2f0d2e77ac1e90495c Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:24:46 +0300 Subject: [PATCH 19/82] Update run-gptq-config-roihu.sh --- GPTQ/run-gptq-config-roihu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/run-gptq-config-roihu.sh b/GPTQ/run-gptq-config-roihu.sh index c9e10f7..f6ebd89 100644 --- a/GPTQ/run-gptq-config-roihu.sh +++ b/GPTQ/run-gptq-config-roihu.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --argos=no -#SBATCH --account=project_2014553 +#SBATCH --account=project_xxxxxxx #SBATCH --partition=gpumedium #SBATCH --time=00:30:00 #SBATCH --nodes=1 From a359378dd7fb3ff1c152ea4eeea49a5169130c11 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:25:01 +0300 Subject: [PATCH 20/82] Update run-gptq-modifier-roihu.sh --- GPTQ/run-gptq-modifier-roihu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/run-gptq-modifier-roihu.sh b/GPTQ/run-gptq-modifier-roihu.sh index 25654ca..e95a826 100644 --- a/GPTQ/run-gptq-modifier-roihu.sh +++ b/GPTQ/run-gptq-modifier-roihu.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --argos=no -#SBATCH --account=project_2014553 +#SBATCH --account=project_xxxxxxx #SBATCH --partition=gpumedium #SBATCH --time=00:30:00 #SBATCH --nodes=1 From 8f1b71bb238c8a15eac08eb4a7c5f9bb587f5029 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:25:45 +0300 Subject: [PATCH 21/82] Update README.md --- AWQ/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AWQ/README.md b/AWQ/README.md index f9c0496..47e0078 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -30,7 +30,8 @@ The flag --cache-dir points the pip cache to the current (scratch) folder instea The launch scripts are: -- `run-awq-modifier-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-awq-modifier-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-awq-modifier-roihu.sh` - quantizes model on Roihu with 1 GPU - `run-awq-modifier-mahti.sh` - quantizes model on Mahti with 1 GPU - `run-awq-modifier-puhti.sh` - quantizes model on Puhti with 1 GPU From 04f6d5b26a720ccfae3295c939fd3a814317bbbb Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:26:18 +0300 Subject: [PATCH 22/82] Update README.md --- BitsAndBytes/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/BitsAndBytes/README.md b/BitsAndBytes/README.md index 86d0443..9aa6ef9 100644 --- a/BitsAndBytes/README.md +++ b/BitsAndBytes/README.md @@ -10,7 +10,8 @@ The script `bnb-quantization.py` will quantize the OPT-125M model to nf4 or Norm The launch scripts are: -- `run-bnb-quantization-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-bnb-quantization-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-bnb-quantization-roihu.sh` - quantizes model on Roihu with 1 GPU - `run-bnb-quantization-mahti.sh` - quantizes model on Mahti with 1 GPU - `run-bnb-quantization-puhti.sh` - quantizes model on Puhti with 1 GPU From 8e3df107eeac10280c3271e0033505b366a1b937 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:26:59 +0300 Subject: [PATCH 23/82] Update README.md --- GPTQ/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 9aa3b93..1e3bd89 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -121,13 +121,15 @@ pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache The launch scripts for gptq-config are: -- `run-gptq-config-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-gptq-config-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-gptq-config-roihu.sh` - quantizes model on Roihu with 1 GPU - `run-gptq-config-mahti.sh` - quantizes model on Mahti with 1 GPU - `run-gptq-config-puhti.sh` - quantizes model on Puhti with 1 GPU Similarly, for gptq-modifier: -- `run-gptq-modifier-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-gptq-modifier-lumi.sh` - quantizes model on LUMI with 1 GPU +- `run-gptq-modifier-roihu.sh` - quantizes model on Roihu with 1 GPU - `run-gptq-modifier-mahti.sh` - quantizes model on Mahti with 1 GPU - `run-gptq-modifier-puhti.sh` - quantizes model on Puhti with 1 GPU From 11e004308dddcb80d35fd6ff3fc961462d15381d Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:29:04 +0300 Subject: [PATCH 24/82] Update README.md --- AWQ/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AWQ/README.md b/AWQ/README.md index 47e0078..fe8c4e6 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -37,6 +37,15 @@ The launch scripts are: **Note:** the scripts are made to be run on `gputest` or `dev-g` partition with a 30 minutes time-limit. You have to select the proper partition for longer jobs for your real runs. Additionally, change the `--account` parameter to your own project code. +**ON ROIHU:** + +**Note:** After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. + +To uninstall: +```bash +(venv)> pip uninstall torch -y +``` + For example to run on LUMI, you would run the command: ```bash From e016eb160a0be7f3764a936088a47c44eb365d5d Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:39:22 +0300 Subject: [PATCH 25/82] Update README.md --- AWQ/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/AWQ/README.md b/AWQ/README.md index fe8c4e6..30f3749 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -6,6 +6,32 @@ In order to target weight and activation scaling locations within the model, the ## Installations +## LUMI + +To run gptq scripts on LUMI, you have to setup a Python environment using `optimum`, `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. + +Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. + +```bash +module purge +module use /appl/local/laifs/modules +module load lumi-aif-singularity-bindings + +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif + +singularity shell "$SIF" + +Singularity> python -m venv venv --system-site-packages +Singularity> source venv/bin/activate +(venv) Singularity> pip install optimum==1.27.0 +(venv) Singularity> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache + +``` +The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. + + +--- + The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples (torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. From 3e23c35b0080dae38b31bc4f10fcc6058f87a990 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:40:15 +0300 Subject: [PATCH 26/82] Update README.md --- GPTQ/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 1e3bd89..2cbaf74 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -68,9 +68,6 @@ To uninstall: (venv)> pip uninstall torch -y ``` - - - --- ## Puhti & Mahti From bb2f7b4a9f4540039d700e8cca199be906bc7ed2 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:41:44 +0300 Subject: [PATCH 27/82] Update README.md --- AWQ/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/AWQ/README.md b/AWQ/README.md index 30f3749..b1990d9 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -32,6 +32,44 @@ The flag --cache-dir points the pip cache to the current (scratch) folder instea --- +## Roihu + +The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples +(torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. + +### Load the module +```bash +module purge +module load python-pytorch/2.10 +``` +### Create and activate a virtual environment using system packages +```bash +python3 -m venv --system-site-packages venv +source venv/bin/activate +``` +### Install packages +```bash +(venv)> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +``` +This version of gptqmodel is compatible with python-pytorch/2.10. + +For the **gptq-modifier** example, you need to install the llmcompressor library. + +```bash +(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install wandb --cache-dir ./.pip-cache +``` + +Note: After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. + +To uninstall: +```bash +(venv)> pip uninstall torch -y +``` +--- + +## Puhti & Mahti + The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples (torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. From bd40b09adaf3323888b9b67597a59491e7284289 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:42:31 +0300 Subject: [PATCH 28/82] Update README.md --- AWQ/README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index b1990d9..bbe70b0 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -101,15 +101,6 @@ The launch scripts are: **Note:** the scripts are made to be run on `gputest` or `dev-g` partition with a 30 minutes time-limit. You have to select the proper partition for longer jobs for your real runs. Additionally, change the `--account` parameter to your own project code. -**ON ROIHU:** - -**Note:** After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. - -To uninstall: -```bash -(venv)> pip uninstall torch -y -``` - For example to run on LUMI, you would run the command: ```bash From 48c378374823f874ae24fbf5f7049d0e186f791e Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:43:38 +0300 Subject: [PATCH 29/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 2cbaf74..5194b1c 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -32,7 +32,7 @@ Singularity> source venv/bin/activate ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. --- +--- ## Roihu The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples From 1284a49edfe4fc07ed2a3fac393fe70e954ac282 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:44:21 +0300 Subject: [PATCH 30/82] Update README.md --- GPTQ/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 5194b1c..6594154 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -89,7 +89,7 @@ source venv/bin/activate ``` ### Install packages ```bash -pip install optimum==1.27.0 --cache-dir ./.pip-cache +(venv)> pip install optimum==1.27.0 --cache-dir ./.pip-cache ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. @@ -100,7 +100,7 @@ The GPTQmodel library is needed for the **gptq-config** example. To install it o Then install with: ```bash -pip install gptqmodel==4.0.0 --no-build-isolation --cache-dir ./.pip-cache +(venv)> pip install gptqmodel==4.0.0 --no-build-isolation --cache-dir ./.pip-cache ``` This version of gptqmodel is compatible with PyTorch 2.7. @@ -112,7 +112,7 @@ Troubleshooting: For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash -pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache ``` ## Usage From 680959d7258aad8d9e0d1f1aca884ccc84325149 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 13:44:36 +0300 Subject: [PATCH 31/82] Update README.md --- AWQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AWQ/README.md b/AWQ/README.md index bbe70b0..ce7e3d2 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -86,7 +86,7 @@ source venv/bin/activate ``` ### Install packages ```bash -pip install optimum==1.27.0 llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install optimum==1.27.0 llmcompressor==0.7.1 --cache-dir ./.pip-cache ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. From 298185bec262a2bb445e4e9509b01f2c043d3b66 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 15:25:36 +0300 Subject: [PATCH 32/82] Update README.md --- GPTQ/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 6594154..8aba367 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -50,15 +50,15 @@ source venv/bin/activate ``` ### Install packages ```bash -(venv)> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +(venv)> pip install gptqmodel==7.1.0 dataset --no-deps --no-build-isolation --cache-dir ./.pip-cache ``` This version of gptqmodel is compatible with python-pytorch/2.10. For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash -(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache -(venv)> pip install wandb --cache-dir ./.pip-cache +(venv)> pip install llmcompressor==0.7.1 --no-deps --cache-dir ./.pip-cache +(venv)> pip install wandb --no-deps --cache-dir ./.pip-cache ``` Note: After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. From 1db80cc546294d3ff2bffa9655006b634a58fa8a Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 15:40:55 +0300 Subject: [PATCH 33/82] Update README.md --- GPTQ/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 8aba367..6594154 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -50,15 +50,15 @@ source venv/bin/activate ``` ### Install packages ```bash -(venv)> pip install gptqmodel==7.1.0 dataset --no-deps --no-build-isolation --cache-dir ./.pip-cache +(venv)> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache ``` This version of gptqmodel is compatible with python-pytorch/2.10. For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash -(venv)> pip install llmcompressor==0.7.1 --no-deps --cache-dir ./.pip-cache -(venv)> pip install wandb --no-deps --cache-dir ./.pip-cache +(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install wandb --cache-dir ./.pip-cache ``` Note: After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. From c28f474aa8a56ecf3bc46165c8678251a038b036 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 15:53:47 +0300 Subject: [PATCH 34/82] Update README.md --- GPTQ/README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 6594154..6dccd88 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -57,17 +57,10 @@ This version of gptqmodel is compatible with python-pytorch/2.10. For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash -(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache (venv)> pip install wandb --cache-dir ./.pip-cache ``` -Note: After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. - -To uninstall: -```bash -(venv)> pip uninstall torch -y -``` - --- ## Puhti & Mahti From d577020bc78688f260ca1d510fa1da46185b7b4d Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 17 Jul 2026 15:54:48 +0300 Subject: [PATCH 35/82] Update README.md --- AWQ/README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index ce7e3d2..93d8092 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -56,16 +56,9 @@ This version of gptqmodel is compatible with python-pytorch/2.10. For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash -(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +(venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache (venv)> pip install wandb --cache-dir ./.pip-cache ``` - -Note: After installing packages, double-check that `torch` and `torchvision` are still coming from the module (`/usr/local/lib64/python3.12/site-packages`), not from your venv. Some packages you install (like `gptqmodel` or `llmcompressor`) can silently pull in their own copy of `torch` into the venv as a side effect — and since a venv-local package always takes priority over the module's version, this creates a mismatched `torch`/`torchvision` pair that breaks imports. - -To uninstall: -```bash -(venv)> pip uninstall torch -y -``` --- ## Puhti & Mahti From 064b7595254d5cd80583575ae5a24121296685f1 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Mon, 20 Jul 2026 16:04:36 +0300 Subject: [PATCH 36/82] Update run-gptq-config-roihu.sh --- GPTQ/run-gptq-config-roihu.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/GPTQ/run-gptq-config-roihu.sh b/GPTQ/run-gptq-config-roihu.sh index f6ebd89..8853f11 100644 --- a/GPTQ/run-gptq-config-roihu.sh +++ b/GPTQ/run-gptq-config-roihu.sh @@ -1,25 +1,21 @@ #!/bin/bash -#SBATCH --argos=no #SBATCH --account=project_xxxxxxx -#SBATCH --partition=gpumedium -#SBATCH --time=00:30:00 +#SBATCH --partition=gputest +#SBATCH --time=00:15:00 #SBATCH --nodes=1 #SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=288 -#SBATCH --gres=gpu:gh200:4 -#SBATCH --mem=240G +#SBATCH --cpus-per-task=72 +#SBATCH --gres=gpu:gh200:1 +#SBATCH --mem=32G + module purge module load python-pytorch/2.10 # Activate the virtual environment from your current directory or change to the appropriate path source venv/bin/activate -# We are putting the cache in the ramdisk, stored in -# memory. Alternatively store it to the project's scratch. -#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ -export HF_HOME=/dev/shm/$USER/hf-cache -export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +# Set hf cache to the project's scratch +export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ mkdir -p $HF_HOME srun python3 gptq-config-roihu.py - From 0fc2524ffabf7acbee3b945de6bf1874a03a8544 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 16:05:46 +0300 Subject: [PATCH 37/82] Delete GPTQ/gptq-config.py --- GPTQ/gptq-config.py | 104 -------------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 GPTQ/gptq-config.py diff --git a/GPTQ/gptq-config.py b/GPTQ/gptq-config.py deleted file mode 100644 index 628ccf6..0000000 --- a/GPTQ/gptq-config.py +++ /dev/null @@ -1,104 +0,0 @@ -import os -import time -import torch -from transformers import ( - AutoTokenizer, - AutoModelForCausalLM, - GPTQConfig, -) - -model_name = "facebook/opt-125m" -prompt = "The future of AI is" - -# Measure model inference time and generate sample output for a given prompt -def benchmark(model, tokenizer, prompt, max_new_tokens=50): - inputs = tokenizer(prompt, return_tensors="pt").to(model.device) - - # Warm-up run (to remove cold start effects) - with torch.no_grad(): - _ = model.generate(**inputs, max_new_tokens=5) - - if torch.cuda.is_available(): - torch.cuda.synchronize() - start = time.time() - - with torch.no_grad(): - output_ids = model.generate( - **inputs, - max_new_tokens=50, - do_sample=True, - temperature=0.7, -) - - if torch.cuda.is_available(): - torch.cuda.synchronize() - end = time.time() - - elapsed_time = end - start - decoded_text = tokenizer.decode(output_ids[0], skip_special_tokens=True) - - return decoded_text, elapsed_time - -# Load base model and tokenizer -model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto") -tokenizer = AutoTokenizer.from_pretrained(model_name) - -# Run benchmark on full model -initial_output, initial_time = benchmark(model, tokenizer, prompt) - -# Save full model before quantization -save_dir_full = model_name.split("/")[-1] + "-full" -model.save_pretrained(save_dir_full, safe_serialization=True) -tokenizer.save_pretrained(save_dir_full) - -# Set GPTQ-config -# Optionally set 'model_seqlen' for models -# where GPTQ cannot automatically infer the max sequence length -gptq_config = GPTQConfig( - bits=4, - dataset="c4", # Use a standard text dataset for calibration - tokenizer=tokenizer - # model_seqlen (int, optional) — The maximum sequence length that the model can take. -) - -# Quantize the model with GPTQ -quantized_model = AutoModelForCausalLM.from_pretrained( - model_name, - quantization_config=gptq_config, - device_map="auto") - -# Save quantized model -save_dir_quant = model_name.split("/")[-1] + "-gptq-config" -# Move model to a CPU for saving -quantized_model.to("cpu") -quantized_model.save_pretrained(save_dir_quant, safe_serialization=True) -tokenizer.save_pretrained(save_dir_quant) - -# Reload quantized model and tokenizer -quant_model = AutoModelForCausalLM.from_pretrained(save_dir_quant, device_map="auto") -quant_tokenizer = AutoTokenizer.from_pretrained(save_dir_quant) - -# Run benchmark on quantized model -quant_output, quant_time = benchmark(quant_model, quant_tokenizer, prompt) - -# Compare model sizes -def get_folder_size(path): - total = 0 - for dirpath, _, filenames in os.walk(path): - for f in filenames: - total += os.path.getsize(os.path.join(dirpath, f)) - return total / (1024 * 1024) # MB - -initial_size = get_folder_size(save_dir_full) -quant_size = get_folder_size(save_dir_quant) - -# Print results -print("=== Full Model ===") -print(f" Output: {initial_output}") -print(f" Size: {initial_size:.2f} MB") -print(f" Inference time: {initial_time:.4f} s") - -print("\n=== Quantized Model ===") -print(f" Output: {quant_output}") -print(f" Size: {quant_size:.2f} MB") -print(f" Inference time: {quant_time:.4f} s") From 214f7a8e88bb2d8e1a6ebb68404f2ed6c536c408 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 16:05:59 +0300 Subject: [PATCH 38/82] Rename gptq-config-roihu.py to gptq-config.py --- GPTQ/{gptq-config-roihu.py => gptq-config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename GPTQ/{gptq-config-roihu.py => gptq-config.py} (100%) diff --git a/GPTQ/gptq-config-roihu.py b/GPTQ/gptq-config.py similarity index 100% rename from GPTQ/gptq-config-roihu.py rename to GPTQ/gptq-config.py From d543c4b11a834756f75d9c1eb6d63d6d5d46a82c Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 16:06:15 +0300 Subject: [PATCH 39/82] Update run-gptq-config-roihu.sh --- GPTQ/run-gptq-config-roihu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/run-gptq-config-roihu.sh b/GPTQ/run-gptq-config-roihu.sh index 8853f11..865e5e2 100644 --- a/GPTQ/run-gptq-config-roihu.sh +++ b/GPTQ/run-gptq-config-roihu.sh @@ -18,4 +18,4 @@ source venv/bin/activate export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ mkdir -p $HF_HOME -srun python3 gptq-config-roihu.py +srun python3 gptq-config.py From 9103d3a5e95bb806f95162d7bff5ea908fd1ba95 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 16:22:07 +0300 Subject: [PATCH 40/82] Update run-bnb-quantization-roihu.sh --- BitsAndBytes/run-bnb-quantization-roihu.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/BitsAndBytes/run-bnb-quantization-roihu.sh b/BitsAndBytes/run-bnb-quantization-roihu.sh index 5facd27..e96799c 100644 --- a/BitsAndBytes/run-bnb-quantization-roihu.sh +++ b/BitsAndBytes/run-bnb-quantization-roihu.sh @@ -1,22 +1,21 @@ #!/bin/bash -#SBATCH --argos=no #SBATCH --account=project_xxxxxxx -#SBATCH --partition=gpumedium -#SBATCH --time=00:30:00 +#SBATCH --partition=gputest +#SBATCH --time=00:15:00 #SBATCH --nodes=1 #SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=288 -#SBATCH --gres=gpu:gh200:4 -#SBATCH --mem=240G +#SBATCH --cpus-per-task=72 +#SBATCH --gres=gpu:gh200:1 +#SBATCH --mem=32G module purge module load python-pytorch/2.10 -# We are putting the cache in the ramdisk, stored in memory. Alternatively store it to the project's scratch. -#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ -export HF_HOME=/dev/shm/$USER/hf-cache -export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +# Activate the virtual environment from your current directory or change to the appropriate path +source venv/bin/activate +# Set hf cache to the project's scratch +export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ mkdir -p $HF_HOME srun python3 bnb-quantization.py From 9a6638e07c13be8d525fd2862b82e28bfd7b27e7 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 16:22:25 +0300 Subject: [PATCH 41/82] Update run-gptq-modifier-roihu.sh --- GPTQ/run-gptq-modifier-roihu.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/GPTQ/run-gptq-modifier-roihu.sh b/GPTQ/run-gptq-modifier-roihu.sh index e95a826..0869a8b 100644 --- a/GPTQ/run-gptq-modifier-roihu.sh +++ b/GPTQ/run-gptq-modifier-roihu.sh @@ -1,24 +1,21 @@ #!/bin/bash -#SBATCH --argos=no #SBATCH --account=project_xxxxxxx -#SBATCH --partition=gpumedium -#SBATCH --time=00:30:00 +#SBATCH --partition=gputest +#SBATCH --time=00:15:00 #SBATCH --nodes=1 #SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=288 -#SBATCH --gres=gpu:gh200:4 -#SBATCH --mem=240G +#SBATCH --cpus-per-task=72 +#SBATCH --gres=gpu:gh200:1 +#SBATCH --mem=32G + module purge module load python-pytorch/2.10 # Activate the virtual environment from your current directory or change to the appropriate path source venv/bin/activate -# We are putting the cache in the ramdisk, stored in -# memory. Alternatively store it to the project's scratch. -#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ -export HF_HOME=/dev/shm/$USER/hf-cache -export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +# Set hf cache to the project's scratch +export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ mkdir -p $HF_HOME srun python3 gptq-modifier.py From 6957f5c98f177c6845d0ad60725c73ec8be757ce Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 16:22:39 +0300 Subject: [PATCH 42/82] Update run-awq-modifier-roihu.sh --- AWQ/run-awq-modifier-roihu.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/AWQ/run-awq-modifier-roihu.sh b/AWQ/run-awq-modifier-roihu.sh index 0922b9b..eeeb5e1 100644 --- a/AWQ/run-awq-modifier-roihu.sh +++ b/AWQ/run-awq-modifier-roihu.sh @@ -1,25 +1,21 @@ #!/bin/bash -#SBATCH --argos=no #SBATCH --account=project_xxxxxxx -#SBATCH --partition=gpumedium -#SBATCH --time=00:30:00 +#SBATCH --partition=gputest +#SBATCH --time=00:15:00 #SBATCH --nodes=1 #SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=288 -#SBATCH --gres=gpu:gh200:4 -#SBATCH --mem=240G +#SBATCH --cpus-per-task=72 +#SBATCH --gres=gpu:gh200:1 +#SBATCH --mem=32G module purge module load python-pytorch/2.10 # Activate the virtual environment from your current directory or change to the appropriate path -source bbvenv/bin/activate +source venv/bin/activate -# We are putting the cache in the ramdisk, stored in -# memory. Alternatively store it to the project's scratch. -#export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ -export HF_HOME=/dev/shm/$USER/hf-cache -export TORCHINDUCTOR_CACHE_DIR=/dev/shm/$USER/ +# Set hf cache to the project's scratch +export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/hf-cache/ mkdir -p $HF_HOME srun python3 awq-modifier.py From 63f63e500d83beb39c7e3784279eb7061bbec52b Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 17:13:44 +0300 Subject: [PATCH 43/82] Update README.md --- GPTQ/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 6dccd88..8210f4e 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -3,7 +3,7 @@ This repository contains two practical examples of applying GPTQ quantization to LLMs. Both examples currently use the small **OPT-125M** model for demonstration, but the code is written so you can swap in larger models. -1. **GPTQConfig** — Uses Hugging Face `transformers` and [`GPTQConfig`](https://huggingface.co/docs/transformers/en/quantization/gptq) to quantize the **OPT-125M** model to 4-bit precision. +1. **GPTQModel** — Uses [GPTQModel](https://github.com/modelcloud/gptqmodel) with `QuantizeConfig` to quantize the **OPT-125M** model to 4-bit precision. 2. **GPTQModifier** — Uses [LLM Compressor](https://github.com/vllm-project/llm-compressor) with a GPTQ recipe to quantize the **OPT-125M** model to mixed precision W4A16. --- @@ -133,7 +133,8 @@ sbatch run-gptq-config-lumi.sh You can also increase the memory if you decide to run quantization on larger models. Setting `device_map="auto"` automatically offloads the model to a CPU to help fit the model in memory, and allow the model modules to be moved between the CPU and GPU for quantization. ## `gptq-config.py` -- Uses Hugging Face 🤗 `transformers` with [`GPTQConfig`](https://huggingface.co/docs/transformers/en/quantization/gptq). +- Uses [`gptqmodel`](https://github.com/modelcloud/gptqmodel)(`GPTQModel.load`, `QuantizeConfig`) with `QuantizeConfig` recipe. +- - Runs explicit **calibration** on a subset of the [aleenai/c4](https://huggingface.co/datasets/allenai/c4) dataset. - This example quantizes the model to 4-bit precision, supported precisions are 2-bit, 3-bit*, 4-bit and 8-bit. - Saves both the full-precision and quantized models. - Compares outputs, inference latency, and model size. From 7fa2ecbdaf6183b09b2b6b4fce202a13b56a7324 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 20 Jul 2026 17:14:14 +0300 Subject: [PATCH 44/82] Update README.md --- GPTQ/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 8210f4e..9fa99c8 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -133,8 +133,8 @@ sbatch run-gptq-config-lumi.sh You can also increase the memory if you decide to run quantization on larger models. Setting `device_map="auto"` automatically offloads the model to a CPU to help fit the model in memory, and allow the model modules to be moved between the CPU and GPU for quantization. ## `gptq-config.py` -- Uses [`gptqmodel`](https://github.com/modelcloud/gptqmodel)(`GPTQModel.load`, `QuantizeConfig`) with `QuantizeConfig` recipe. -- - Runs explicit **calibration** on a subset of the [aleenai/c4](https://huggingface.co/datasets/allenai/c4) dataset. +- Uses [`gptqmodel`](https://github.com/modelcloud/gptqmodel)(`GPTQModel.load`, `QuantizeConfig`) with `QuantizeConfig` recipe. +- Runs explicit **calibration** on a subset of the [aleenai/c4](https://huggingface.co/datasets/allenai/c4) dataset. - This example quantizes the model to 4-bit precision, supported precisions are 2-bit, 3-bit*, 4-bit and 8-bit. - Saves both the full-precision and quantized models. - Compares outputs, inference latency, and model size. From d51b9083c681b81323c889ee3e78129d91ef4c83 Mon Sep 17 00:00:00 2001 From: anni-moisala Date: Tue, 21 Jul 2026 16:31:38 +0300 Subject: [PATCH 45/82] remove puhti and mahti scripts --- BitsAndBytes/run-bnb-quantization-mahti.sh | 23 ---------------------- BitsAndBytes/run-bnb-quantization-puhti.sh | 23 ---------------------- 2 files changed, 46 deletions(-) delete mode 100644 BitsAndBytes/run-bnb-quantization-mahti.sh delete mode 100644 BitsAndBytes/run-bnb-quantization-puhti.sh diff --git a/BitsAndBytes/run-bnb-quantization-mahti.sh b/BitsAndBytes/run-bnb-quantization-mahti.sh deleted file mode 100644 index 6c683e7..0000000 --- a/BitsAndBytes/run-bnb-quantization-mahti.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:a100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 bnb-quantization.py diff --git a/BitsAndBytes/run-bnb-quantization-puhti.sh b/BitsAndBytes/run-bnb-quantization-puhti.sh deleted file mode 100644 index 30d4626..0000000 --- a/BitsAndBytes/run-bnb-quantization-puhti.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:v100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 bnb-quantization.py From 15873774cedeb32833ef3bfe44403ef632eefa6c Mon Sep 17 00:00:00 2001 From: anni-moisala Date: Tue, 21 Jul 2026 16:41:44 +0300 Subject: [PATCH 46/82] remove remaining puhti and mahti scripts --- AWQ/run-awq-modifier-mahti.sh | 26 -------------------------- AWQ/run-awq-modifier-puhti.sh | 26 -------------------------- GPTQ/run-gptq-config-mahti.sh | 26 -------------------------- GPTQ/run-gptq-config-puhti.sh | 26 -------------------------- GPTQ/run-gptq-modifier-mahti.sh | 26 -------------------------- GPTQ/run-gptq-modifier-puhti.sh | 26 -------------------------- 6 files changed, 156 deletions(-) delete mode 100644 AWQ/run-awq-modifier-mahti.sh delete mode 100644 AWQ/run-awq-modifier-puhti.sh delete mode 100644 GPTQ/run-gptq-config-mahti.sh delete mode 100644 GPTQ/run-gptq-config-puhti.sh delete mode 100644 GPTQ/run-gptq-modifier-mahti.sh delete mode 100644 GPTQ/run-gptq-modifier-puhti.sh diff --git a/AWQ/run-awq-modifier-mahti.sh b/AWQ/run-awq-modifier-mahti.sh deleted file mode 100644 index d131122..0000000 --- a/AWQ/run-awq-modifier-mahti.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:a100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 awq-modifier.py diff --git a/AWQ/run-awq-modifier-puhti.sh b/AWQ/run-awq-modifier-puhti.sh deleted file mode 100644 index 1e40cff..0000000 --- a/AWQ/run-awq-modifier-puhti.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:v100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 awq-modifier.py diff --git a/GPTQ/run-gptq-config-mahti.sh b/GPTQ/run-gptq-config-mahti.sh deleted file mode 100644 index a9af25a..0000000 --- a/GPTQ/run-gptq-config-mahti.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:a100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 gptq-config.py diff --git a/GPTQ/run-gptq-config-puhti.sh b/GPTQ/run-gptq-config-puhti.sh deleted file mode 100644 index 70f7961..0000000 --- a/GPTQ/run-gptq-config-puhti.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:v100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 gptq-config.py diff --git a/GPTQ/run-gptq-modifier-mahti.sh b/GPTQ/run-gptq-modifier-mahti.sh deleted file mode 100644 index 7d8b51f..0000000 --- a/GPTQ/run-gptq-modifier-mahti.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:a100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 gptq-modifier.py diff --git a/GPTQ/run-gptq-modifier-puhti.sh b/GPTQ/run-gptq-modifier-puhti.sh deleted file mode 100644 index 41abf50..0000000 --- a/GPTQ/run-gptq-modifier-puhti.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --account=xxxxxxxx -#SBATCH --partition=gputest -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=6 -#SBATCH --nodes=1 -#SBATCH --mem=32G -#SBATCH --time=0:15:00 -#SBATCH --gres=gpu:v100:1 -#SBATCH --output=slurm-%j.out -#SBATCH --error=slurm-%j.err - -# Load the module -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 - -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - -# This will store all the Hugging Face cache such as downloaded models -# and datasets in the project's scratch folder -export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache -mkdir -p $HF_HOME - -srun python3 gptq-modifier.py From 396d836142d5bede64bbd258d91cef618fac366a Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Thu, 23 Jul 2026 12:59:38 +0300 Subject: [PATCH 47/82] Update README.md --- GPTQ/README.md | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 9fa99c8..4034f53 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -58,55 +58,10 @@ For the **gptq-modifier** example, you need to install the llmcompressor library ```bash (venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache -(venv)> pip install wandb --cache-dir ./.pip-cache ``` --- -## Puhti & Mahti -### Installations - -The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples -(torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. - -### Load the module -```bash -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 -``` -### Create and activate a virtual environment using system packages -```bash -python3 -m venv --system-site-packages venv -source venv/bin/activate -``` -### Install packages -```bash -(venv)> pip install optimum==1.27.0 --cache-dir ./.pip-cache -``` -The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. - -The GPTQmodel library is needed for the **gptq-config** example. To install it on Puhti or Mahti, you need to use a GPU interactively when installing, or set the following environment variable: - -- For Puhti:`export TORCH_CUDA_ARCH_LIST="7.0"` -- For Mahti: `export TORCH_CUDA_ARCH_LIST="8.0"` - -Then install with: -```bash -(venv)> pip install gptqmodel==4.0.0 --no-build-isolation --cache-dir ./.pip-cache -``` - -This version of gptqmodel is compatible with PyTorch 2.7. - -Troubleshooting: -- If you get an AssetionError from pip's resolver when installing gptqmodel, upgrade pip, setuptools and wheel: `python -m pip install --upgrade pip setuptools wheel` -- When quantizing models that use Rotary Positional Embeddings (RoPE), such as LlaMA, you might encounter runtime errors related to rotary dimensions. The current fix is to downgrade transformers to version 4.51.3. - -For the **gptq-modifier** example, you need to install the llmcompressor library. - -```bash -(venv)> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache -``` ## Usage The launch scripts for gptq-config are: From b20e7fd308a7df01ed73acf570fb666d96f5d4c5 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Thu, 23 Jul 2026 13:00:00 +0300 Subject: [PATCH 48/82] Update README.md --- BitsAndBytes/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/BitsAndBytes/README.md b/BitsAndBytes/README.md index 9aa6ef9..cab6fb4 100644 --- a/BitsAndBytes/README.md +++ b/BitsAndBytes/README.md @@ -12,8 +12,6 @@ The launch scripts are: - `run-bnb-quantization-lumi.sh` - quantizes model on LUMI with 1 GPU - `run-bnb-quantization-roihu.sh` - quantizes model on Roihu with 1 GPU -- `run-bnb-quantization-mahti.sh` - quantizes model on Mahti with 1 GPU -- `run-bnb-quantization-puhti.sh` - quantizes model on Puhti with 1 GPU **Note:** the scripts are made to be run on `gputest` or `dev-g` partition with a 30 minutes time-limit. You have to select the proper partition for longer jobs for your real runs. Additionally, change the `--account` parameter to your own project code. From b91ec83c3cae1da23c8344e68fc8986a9148ff41 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Thu, 23 Jul 2026 13:00:22 +0300 Subject: [PATCH 49/82] Update README.md --- GPTQ/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 4034f53..2d5fd6b 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -68,15 +68,11 @@ The launch scripts for gptq-config are: - `run-gptq-config-lumi.sh` - quantizes model on LUMI with 1 GPU - `run-gptq-config-roihu.sh` - quantizes model on Roihu with 1 GPU -- `run-gptq-config-mahti.sh` - quantizes model on Mahti with 1 GPU -- `run-gptq-config-puhti.sh` - quantizes model on Puhti with 1 GPU Similarly, for gptq-modifier: - `run-gptq-modifier-lumi.sh` - quantizes model on LUMI with 1 GPU - `run-gptq-modifier-roihu.sh` - quantizes model on Roihu with 1 GPU -- `run-gptq-modifier-mahti.sh` - quantizes model on Mahti with 1 GPU -- `run-gptq-modifier-puhti.sh` - quantizes model on Puhti with 1 GPU **Note:** the scripts are made to be run on `gputest` or `dev-g` partition with a 30 minute time-limit. You have to select the proper partition for longer jobs for your real runs. Additionally, change the `--account` parameter to your own project code. From 15cf172a4fd874d9ab6773baf49f751a1a695342 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Thu, 23 Jul 2026 13:00:40 +0300 Subject: [PATCH 50/82] Update README.md --- AWQ/README.md | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index 93d8092..0d70e46 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -61,36 +61,12 @@ For the **gptq-modifier** example, you need to install the llmcompressor library ``` --- -## Puhti & Mahti - -The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples -(torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. - -### Load the module -```bash -module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 -``` -### Create and activate a virtual environment using system packages -```bash -python3 -m venv --system-site-packages venv -source venv/bin/activate -``` -### Install packages -```bash -(venv)> pip install optimum==1.27.0 llmcompressor==0.7.1 --cache-dir ./.pip-cache -``` -The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. - ## Usage The launch scripts are: - `run-awq-modifier-lumi.sh` - quantizes model on LUMI with 1 GPU - `run-awq-modifier-roihu.sh` - quantizes model on Roihu with 1 GPU -- `run-awq-modifier-mahti.sh` - quantizes model on Mahti with 1 GPU -- `run-awq-modifier-puhti.sh` - quantizes model on Puhti with 1 GPU **Note:** the scripts are made to be run on `gputest` or `dev-g` partition with a 30 minutes time-limit. You have to select the proper partition for longer jobs for your real runs. Additionally, change the `--account` parameter to your own project code. From 5004d3f4e0624361195e5e27249820e47bee0f90 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 24 Jul 2026 13:25:06 +0300 Subject: [PATCH 51/82] Update README.md --- GPTQ/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 2d5fd6b..ed07c6a 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -23,11 +23,10 @@ export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-2026051 singularity shell "$SIF" -Singularity> python -m venv venv --system-site-packages -Singularity> source venv/bin/activate -(venv) Singularity> pip install optimum==1.27.0 -(venv) Singularity> pip install gptqmodel==4.0.0 --no-build-isolation --cache-dir ./.pip-cache -(venv) Singularity> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +Apptainer> python -m venv venv --system-site-packages +Apptainer> source venv/bin/activate +(venv) Apptainer> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +(venv) Apptainer> pip install llmcompressor==0.10.0 --cache-dir ./.pip-cache ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. From 1c593fa5db077a5dc9169fde16052e4ee18d82c0 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 24 Jul 2026 13:25:29 +0300 Subject: [PATCH 52/82] Update README.md --- AWQ/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index 0d70e46..c21bea8 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -21,10 +21,10 @@ export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-2026051 singularity shell "$SIF" -Singularity> python -m venv venv --system-site-packages -Singularity> source venv/bin/activate -(venv) Singularity> pip install optimum==1.27.0 -(venv) Singularity> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache +Apptainer> python -m venv venv --system-site-packages +Apptainer> source venv/bin/activate +(venv) Apptainer> pip install optimum==1.27.0 +(venv) Apptainer> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. From a041cb1b812113d87000ff780cb19da080c717ad Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 24 Jul 2026 13:33:36 +0300 Subject: [PATCH 53/82] Update README.md --- GPTQ/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index ed07c6a..f755feb 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -85,7 +85,7 @@ You can also increase the memory if you decide to run quantization on larger mod ## `gptq-config.py` - Uses [`gptqmodel`](https://github.com/modelcloud/gptqmodel)(`GPTQModel.load`, `QuantizeConfig`) with `QuantizeConfig` recipe. - Runs explicit **calibration** on a subset of the [aleenai/c4](https://huggingface.co/datasets/allenai/c4) dataset. -- This example quantizes the model to 4-bit precision, supported precisions are 2-bit, 3-bit*, 4-bit and 8-bit. +- This example quantizes the model to 4-bit precision, supported precisions are 2-bit, 3-bit, 4-bit and 8-bit. - Saves both the full-precision and quantized models. - Compares outputs, inference latency, and model size. @@ -97,8 +97,6 @@ You can also increase the memory if you decide to run quantization on larger mod - Compares outputs, inference latency, and model size. - Provides finer control over quantization schemes (e.g. `W4A16`, `ignore=["lm_head"]`). -*3-bit quantization is not currently supported on LUMI with PyTorch 2.7, if you wish to use 3-bit quantization you can use the PyTorch 2.5 module. - ## Output Includes - Generated text before and after quantization. - Inference time comparison. From 092c1e87ec53d2ea89267efb26b8e72d2a1317a4 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 24 Jul 2026 13:43:03 +0300 Subject: [PATCH 54/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index f755feb..b53ae51 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -25,7 +25,7 @@ singularity shell "$SIF" Apptainer> python -m venv venv --system-site-packages Apptainer> source venv/bin/activate -(venv) Apptainer> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +(venv) Apptainer> pip install gptqmodel==7.1.0 --no-build-isolation --cache-dir ./.pip-cache (venv) Apptainer> pip install llmcompressor==0.10.0 --cache-dir ./.pip-cache ``` From f0d201e29e77d921dc2772d8bdf17d9ccd2c37e0 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Fri, 24 Jul 2026 13:50:19 +0300 Subject: [PATCH 55/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index b53ae51..8124adc 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -49,7 +49,7 @@ source venv/bin/activate ``` ### Install packages ```bash -(venv)> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache +(venv)> pip install gptqmodel==7.1.0 --no-build-isolation --cache-dir ./.pip-cache ``` This version of gptqmodel is compatible with python-pytorch/2.10. From 0eccd4aecb4660246d7a43f8f9f30b4461f93002 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 27 Jul 2026 12:57:16 +0300 Subject: [PATCH 56/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 8124adc..4465010 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -10,7 +10,7 @@ Both examples currently use the small **OPT-125M** model for demonstration, but ## LUMI -To run gptq scripts on LUMI, you have to setup a Python environment using `optimum`, `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. +To run gptq scripts on LUMI, you have to setup a Python environment using `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. From f771bfc1fecf8e7405a02c051875d775e863a690 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 27 Jul 2026 12:57:26 +0300 Subject: [PATCH 57/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 4465010..88817c7 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -10,7 +10,7 @@ Both examples currently use the small **OPT-125M** model for demonstration, but ## LUMI -To run gptq scripts on LUMI, you have to setup a Python environment using `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. +To run gptq scripts on LUMI, you have to setup a Python environment using `gptqmodel` and `llmcompressor`, built on the top of LUMI's AI Singularity framework. Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. From 42d64bee7ac5e8a54af58db228d74d2076d54231 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 29 Jul 2026 11:59:42 +0300 Subject: [PATCH 58/82] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c80b50..520560f 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,6 @@ We provide examples and experiments for: 3. **Guides & Utilities** - Helper functions for measuring model size, timing inference, and testing the quantized model. - - Notes on how to run the examples on Puhti, Mahti and LUMI. + - Notes on how to run the examples on LUMI and Roihu. --- From b5b3d1bcc0269cb489ba40376d5fe354e88a75d6 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 29 Jul 2026 12:07:35 +0300 Subject: [PATCH 59/82] Update README.md --- GPTQ/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index 88817c7..f299ddd 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -10,9 +10,9 @@ Both examples currently use the small **OPT-125M** model for demonstration, but ## LUMI -To run gptq scripts on LUMI, you have to setup a Python environment using `gptqmodel` and `llmcompressor`, built on the top of LUMI's AI Singularity framework. +To run gptq scripts on LUMI, you have to install `gptqmodel` and `llmcompressor` on top of the LUMI AI Factory container in a virtual environment. -Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. +Load the `Singularity` container environment and set the container image path. Later, create virtual environment inside the container and install the packages. ```bash module purge From 75209650808e09415e7e15f30f889a16cfde95ed Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 29 Jul 2026 12:09:38 +0300 Subject: [PATCH 60/82] fix formatting --- GPTQ/run-gptq-config-lumi.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/GPTQ/run-gptq-config-lumi.sh b/GPTQ/run-gptq-config-lumi.sh index e862660..3396b71 100644 --- a/GPTQ/run-gptq-config-lumi.sh +++ b/GPTQ/run-gptq-config-lumi.sh @@ -17,7 +17,6 @@ module load lumi-aif-singularity-bindings # export path to used container image export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif - # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache From 6d7588419dc64ee5d6054849cce6b7ebe7db4c12 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 29 Jul 2026 12:39:49 +0300 Subject: [PATCH 61/82] Update README.md --- GPTQ/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index f299ddd..7c36316 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -10,7 +10,7 @@ Both examples currently use the small **OPT-125M** model for demonstration, but ## LUMI -To run gptq scripts on LUMI, you have to install `gptqmodel` and `llmcompressor` on top of the LUMI AI Factory container in a virtual environment. +To run gptq scripts on LUMI, you have to install `gptqmodel` on top of the LUMI AI Factory container in a virtual environment. `Llmcompressor` and other libraries needed are already in the container. Load the `Singularity` container environment and set the container image path. Later, create virtual environment inside the container and install the packages. @@ -26,7 +26,6 @@ singularity shell "$SIF" Apptainer> python -m venv venv --system-site-packages Apptainer> source venv/bin/activate (venv) Apptainer> pip install gptqmodel==7.1.0 --no-build-isolation --cache-dir ./.pip-cache -(venv) Apptainer> pip install llmcompressor==0.10.0 --cache-dir ./.pip-cache ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. From f5c84f433d2a9d090351350c482098cb25770f92 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 29 Jul 2026 12:42:17 +0300 Subject: [PATCH 62/82] small fixes --- BitsAndBytes/run-bnb-quantization-lumi.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/BitsAndBytes/run-bnb-quantization-lumi.sh b/BitsAndBytes/run-bnb-quantization-lumi.sh index 86a64b4..e2b726a 100644 --- a/BitsAndBytes/run-bnb-quantization-lumi.sh +++ b/BitsAndBytes/run-bnb-quantization-lumi.sh @@ -7,7 +7,6 @@ #SBATCH --mem=32G #SBATCH --time=00:30:00 - # Load the module module purge module use /appl/local/laifs/modules @@ -18,7 +17,7 @@ export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-2026051 # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder -export HF_HOME=/scratch/project_462001302/mmahnoor/llm-quantization-scripts/BitsAndBytes/hf-cache +export HF_HOME=/scratch/$SLURM_JOB_ACCOUNT/$USER/llm-quantization-scripts/BitsAndBytes/hf-cache mkdir -p $HF_HOME export SINGULARITYENV_HF_HOME=$HF_HOME From 47983f71baf736907a53b89d40aad9efdc4a1896 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 29 Jul 2026 13:56:00 +0300 Subject: [PATCH 63/82] Update README.md --- AWQ/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index c21bea8..30856dc 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -8,9 +8,9 @@ In order to target weight and activation scaling locations within the model, the ## LUMI -To run gptq scripts on LUMI, you have to setup a Python environment using `optimum`, `gptqmodel`, and `llmcompressor`, built on the top of LUMI's AI Singularity framework. +To run gptq scripts on LUMI, you have to install `gptqmodel` on top of the LUMI AI Factory container in a virtual environment. `Llmcompressor` and other libraries needed are already in the container. -Load the `Singularity` container environment and set the contianer image path. Later, create virtual environment inside the contianer and install the packages. +Load the `Singularity` container environment and set the container image path. Later, create virtual environment inside the container and install the packages. ```bash module purge @@ -24,7 +24,6 @@ singularity shell "$SIF" Apptainer> python -m venv venv --system-site-packages Apptainer> source venv/bin/activate (venv) Apptainer> pip install optimum==1.27.0 -(venv) Apptainer> pip install llmcompressor==0.7.1 --cache-dir ./.pip-cache ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. @@ -57,7 +56,6 @@ For the **gptq-modifier** example, you need to install the llmcompressor library ```bash (venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache -(venv)> pip install wandb --cache-dir ./.pip-cache ``` --- From e51b8b978d3b0bb4c5b5d2b912d1c9956a86187b Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 10:37:20 +0300 Subject: [PATCH 64/82] updated script to latest released container --- GPTQ/run-gptq-modifier-lumi.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/GPTQ/run-gptq-modifier-lumi.sh b/GPTQ/run-gptq-modifier-lumi.sh index 2dee594..eca6e9a 100644 --- a/GPTQ/run-gptq-modifier-lumi.sh +++ b/GPTQ/run-gptq-modifier-lumi.sh @@ -11,8 +11,11 @@ # Load the module module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 +module use /appl/local/laifs/modules +module load lumi-aif-singularity-bindings + +# export path to used container image +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif # Activate the virtual environment from your current directory or change to the appropriate path source venv/bin/activate @@ -22,4 +25,4 @@ source venv/bin/activate export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache mkdir -p $HF_HOME -srun python3 gptq-modifier.py +srun singularity exec "$SIF" bash -c 'source venv/bin/activate && python3 gptq-modifier.py' From ace163f0b7687c741e42baa71834ca051ee8dea8 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 10:37:50 +0300 Subject: [PATCH 65/82] minor update --- GPTQ/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/GPTQ/README.md b/GPTQ/README.md index 7c36316..ab5fc88 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -29,6 +29,11 @@ Apptainer> source venv/bin/activate ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. +For the **gptq-modifier** example, you need to install the llmcompressor library. + +```bash +(venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache +``` --- ## Roihu From bfa1cef76e7fac23e59b4a8cec5d6c36fa21c0d1 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 10:38:13 +0300 Subject: [PATCH 66/82] minor update --- GPTQ/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/GPTQ/README.md b/GPTQ/README.md index ab5fc88..d5ecf97 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -29,6 +29,7 @@ Apptainer> source venv/bin/activate ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. + For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash From c8120d9b15ff5d51e6e217f889042aa75c896fa4 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 10:47:42 +0300 Subject: [PATCH 67/82] Update README.md --- GPTQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index d5ecf97..edc264a 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -19,7 +19,7 @@ module purge module use /appl/local/laifs/modules module load lumi-aif-singularity-bindings -export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif singularity shell "$SIF" From fbeff7f1dab148d45b46b8ca4c585333fc67aa88 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 10:48:11 +0300 Subject: [PATCH 68/82] updates container --- GPTQ/run-gptq-config-lumi.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GPTQ/run-gptq-config-lumi.sh b/GPTQ/run-gptq-config-lumi.sh index 3396b71..65fa56a 100644 --- a/GPTQ/run-gptq-config-lumi.sh +++ b/GPTQ/run-gptq-config-lumi.sh @@ -15,7 +15,10 @@ module use /appl/local/laifs/modules module load lumi-aif-singularity-bindings # export path to used container image -export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif + +# Activate the virtual environment from your current directory or change to the appropriate path +source venv/bin/activate # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder From c7e63376039ed004280efb3f4a59adafd3114137 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 10:56:16 +0300 Subject: [PATCH 69/82] updates to container: lumi-multitorch-u24r70f21m50t210-20260731_122833 --- BitsAndBytes/run-bnb-quantization-lumi.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BitsAndBytes/run-bnb-quantization-lumi.sh b/BitsAndBytes/run-bnb-quantization-lumi.sh index e2b726a..5ea4e29 100644 --- a/BitsAndBytes/run-bnb-quantization-lumi.sh +++ b/BitsAndBytes/run-bnb-quantization-lumi.sh @@ -6,6 +6,8 @@ #SBATCH --gpus-per-node=1 #SBATCH --mem=32G #SBATCH --time=00:30:00 +#SBATCH --output=slurm-%j.out +#SBATCH --error=slurm-%j.err # Load the module module purge @@ -13,7 +15,7 @@ module use /appl/local/laifs/modules module load lumi-aif-singularity-bindings # export path to used container image -export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder From f5eea14a245fef98b1abef49aa8a06c971a1cc29 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 11:25:20 +0300 Subject: [PATCH 70/82] minor update --- GPTQ/run-gptq-config-lumi.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/GPTQ/run-gptq-config-lumi.sh b/GPTQ/run-gptq-config-lumi.sh index 65fa56a..da970f8 100644 --- a/GPTQ/run-gptq-config-lumi.sh +++ b/GPTQ/run-gptq-config-lumi.sh @@ -17,9 +17,6 @@ module load lumi-aif-singularity-bindings # export path to used container image export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache From 4c29e29107e3db71f43e438f86b16227dd34866f Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Mon, 10 Aug 2026 11:25:36 +0300 Subject: [PATCH 71/82] minor update --- GPTQ/run-gptq-modifier-lumi.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/GPTQ/run-gptq-modifier-lumi.sh b/GPTQ/run-gptq-modifier-lumi.sh index eca6e9a..dc44772 100644 --- a/GPTQ/run-gptq-modifier-lumi.sh +++ b/GPTQ/run-gptq-modifier-lumi.sh @@ -17,9 +17,6 @@ module load lumi-aif-singularity-bindings # export path to used container image export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate - # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache From ac2d0a036422fe684d0339ee081742757322332e Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 14:05:22 +0300 Subject: [PATCH 72/82] change the model_name Working model: TinyLlama/TinyLlama-1.1B-Chat-v1.0 --- AWQ/awq-modifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AWQ/awq-modifier.py b/AWQ/awq-modifier.py index fe70eea..3ea2cf2 100644 --- a/AWQ/awq-modifier.py +++ b/AWQ/awq-modifier.py @@ -11,7 +11,7 @@ from llmcompressor.modifiers.awq import AWQModifier from llmcompressor.utils import dispatch_for_generation -model_name = "tiiuae/falcon-rw-1b" +model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" dataset_name = "HuggingFaceH4/ultrachat_200k" dataset_split = "train_sft" num_calibration_samples = 256 From 20fcad43b20397cddd10b4dad0548d357f17179f Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 14:06:02 +0300 Subject: [PATCH 73/82] major changes in the script --- AWQ/run-awq-modifier-lumi.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/AWQ/run-awq-modifier-lumi.sh b/AWQ/run-awq-modifier-lumi.sh index 61cfd1c..8b7e9a1 100644 --- a/AWQ/run-awq-modifier-lumi.sh +++ b/AWQ/run-awq-modifier-lumi.sh @@ -11,15 +11,17 @@ # Load the module module purge -module use /appl/local/csc/modulefiles -module load pytorch/2.7 +module use /appl/local/laifs/modules +module load lumi-aif-singularity-bindings -# Activate the virtual environment from your current directory or change to the appropriate path -source venv/bin/activate +# export path to used container image +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50> + +# Activate the virtual environment from your current directory or change to th # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache mkdir -p $HF_HOME -srun python3 awq-modifier.py +srun singularity exec "$SIF" python3 awq-modifier.py From 895adf8a5e2f73761fc47b58a7f8bb3338b3afd6 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 16:06:33 +0300 Subject: [PATCH 74/82] major updates --- AWQ/README.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index 30856dc..0e2ad41 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -8,7 +8,7 @@ In order to target weight and activation scaling locations within the model, the ## LUMI -To run gptq scripts on LUMI, you have to install `gptqmodel` on top of the LUMI AI Factory container in a virtual environment. `Llmcompressor` and other libraries needed are already in the container. +To run AWQ quantization scripts on LUMI, you can use the `gptqmodel` library that already ships inside the LUMI AI Factory container — no extra packages or virtual environment are required for AWQ. `torch` , `transformers`, and `datasets` are also already provided in the container. Load the `Singularity` container environment and set the container image path. Later, create virtual environment inside the container and install the packages. @@ -17,17 +17,9 @@ module purge module use /appl/local/laifs/modules module load lumi-aif-singularity-bindings -export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260513_121430/lumi-multitorch-full-u24r70f21m50t210-20260513_121430.sif - -singularity shell "$SIF" - -Apptainer> python -m venv venv --system-site-packages -Apptainer> source venv/bin/activate -(venv) Apptainer> pip install optimum==1.27.0 +export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50t210-20260731_122833.sif ``` -The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. - --- @@ -47,12 +39,6 @@ python3 -m venv --system-site-packages venv source venv/bin/activate ``` ### Install packages -```bash -(venv)> pip install gptqmodel==7.1.0 dataset --no-build-isolation --cache-dir ./.pip-cache -``` -This version of gptqmodel is compatible with python-pytorch/2.10. - -For the **gptq-modifier** example, you need to install the llmcompressor library. ```bash (venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache From 26d8b32c170cdffc5a42a1da121db1c4a619a3ab Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 16:07:31 +0300 Subject: [PATCH 75/82] minor changes --- AWQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AWQ/README.md b/AWQ/README.md index 0e2ad41..a3e9d1b 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -73,6 +73,6 @@ Meaning the script quantizes the model’s linear layers using a mixed-precision - Model size (MB) before and after quantization. ## Notes -- The current scripts use **Falcon-RW-1B** for fast experimentation. You can replace `model_name` with a larger model. In this case, you might want to disable saving the full model. +- The current scripts use **TinyLlama** for fast experimentation. You can replace `model_name` with a larger model. In this case, you might want to disable saving the full model. - For large models, `device_map="auto"` allows the model modules to be moved between the CPU and GPU for quantization. - Feel free to experiment with different values for `num_calibration_samples` and `max_seq_lenght` and to modify the quantization recipe. From ba105a4892eea5e4c51191105d312ba835ed717b Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 16:08:15 +0300 Subject: [PATCH 76/82] minor update --- AWQ/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AWQ/README.md b/AWQ/README.md index a3e9d1b..5b5a068 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -73,6 +73,6 @@ Meaning the script quantizes the model’s linear layers using a mixed-precision - Model size (MB) before and after quantization. ## Notes -- The current scripts use **TinyLlama** for fast experimentation. You can replace `model_name` with a larger model. In this case, you might want to disable saving the full model. +- The current scripts use **TinyLlama-1.1B-Chat-v1.0** for fast experimentation. You can replace `model_name` with a larger model. In this case, you might want to disable saving the full model. - For large models, `device_map="auto"` allows the model modules to be moved between the CPU and GPU for quantization. - Feel free to experiment with different values for `num_calibration_samples` and `max_seq_lenght` and to modify the quantization recipe. From 56ef3f431ca08aa9b4dfdcfe057aa17fc2323a68 Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 16:08:39 +0300 Subject: [PATCH 77/82] minor update --- AWQ/run-awq-modifier-lumi.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/AWQ/run-awq-modifier-lumi.sh b/AWQ/run-awq-modifier-lumi.sh index 8b7e9a1..dc86524 100644 --- a/AWQ/run-awq-modifier-lumi.sh +++ b/AWQ/run-awq-modifier-lumi.sh @@ -17,7 +17,6 @@ module load lumi-aif-singularity-bindings # export path to used container image export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260731_122833/lumi-multitorch-full-u24r70f21m50> - # Activate the virtual environment from your current directory or change to th # This will store all the Hugging Face cache such as downloaded models # and datasets in the project's scratch folder From e1695af8d1fa1610e27f93a4a570ac0cc544cc8d Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 16:32:51 +0300 Subject: [PATCH 78/82] updated script to remove venv no additional modules required, all required modules are pre installed in SIF. --- GPTQ/run-gptq-modifier-lumi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPTQ/run-gptq-modifier-lumi.sh b/GPTQ/run-gptq-modifier-lumi.sh index dc44772..c033dc0 100644 --- a/GPTQ/run-gptq-modifier-lumi.sh +++ b/GPTQ/run-gptq-modifier-lumi.sh @@ -22,4 +22,4 @@ export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-2026073 export HF_HOME=/scratch/${SLURM_JOB_ACCOUNT}/${USER}/hf-cache mkdir -p $HF_HOME -srun singularity exec "$SIF" bash -c 'source venv/bin/activate && python3 gptq-modifier.py' +srun singularity exec "$SIF" python3 gptq-modifier.py From c7504bc44aa597786bdc7b0b171dd4af9f25402f Mon Sep 17 00:00:00 2001 From: Mahnoor Mahnoor Date: Tue, 11 Aug 2026 16:33:29 +0300 Subject: [PATCH 79/82] updated --- GPTQ/README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/GPTQ/README.md b/GPTQ/README.md index edc264a..c570722 100644 --- a/GPTQ/README.md +++ b/GPTQ/README.md @@ -30,12 +30,6 @@ Apptainer> source venv/bin/activate ``` The flag --cache-dir points the pip cache to the current (scratch) folder instead of the default (home directory), to avoid filling up home directory quota. -For the **gptq-modifier** example, you need to install the llmcompressor library. - -```bash -(venv)> pip install llmcompressor==0.12.0 --cache-dir ./.pip-cache -``` - --- ## Roihu From 0ee1d0e688dc89dad77bf45fe77ae6945dbd0cf3 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 12 Aug 2026 10:33:57 +0300 Subject: [PATCH 80/82] Update README.md --- BitsAndBytes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BitsAndBytes/README.md b/BitsAndBytes/README.md index cab6fb4..6601af1 100644 --- a/BitsAndBytes/README.md +++ b/BitsAndBytes/README.md @@ -4,7 +4,7 @@ This example demonstrates quantizing the **OPT-125M** model using the [bitsandby ## Running the script -All of the libraries needed to run this example (transformers, bitsandbytes, accelerate) are covered by the CSC preinstalled PyTorch module. +All of the libraries needed to run this example (transformers, bitsandbytes, accelerate) are covered by the Lumi AI Factory Container on LUMI or the CSC preinstalled PyTorch module on Roihu. The script `bnb-quantization.py` will quantize the OPT-125M model to nf4 or NormalFloat 4-bit, introduced to use with QLoRA technique, a parameter efficient fine-tuning technique. It can be used with QLoRA for fine-tuning, or without just for reducing model size. From 4ed35d53bb52e36facd09a26dc6b330a1573254b Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 12 Aug 2026 10:34:56 +0300 Subject: [PATCH 81/82] Update README.md --- BitsAndBytes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BitsAndBytes/README.md b/BitsAndBytes/README.md index 6601af1..629b126 100644 --- a/BitsAndBytes/README.md +++ b/BitsAndBytes/README.md @@ -4,7 +4,7 @@ This example demonstrates quantizing the **OPT-125M** model using the [bitsandby ## Running the script -All of the libraries needed to run this example (transformers, bitsandbytes, accelerate) are covered by the Lumi AI Factory Container on LUMI or the CSC preinstalled PyTorch module on Roihu. +All of the libraries needed to run this example (transformers, bitsandbytes, accelerate) are covered by the AI Factory provided Container on LUMI or the CSC preinstalled PyTorch module on Roihu. The script `bnb-quantization.py` will quantize the OPT-125M model to nf4 or NormalFloat 4-bit, introduced to use with QLoRA technique, a parameter efficient fine-tuning technique. It can be used with QLoRA for fine-tuning, or without just for reducing model size. From b2095f5b2733552ae09809d24f455679e8a18579 Mon Sep 17 00:00:00 2001 From: Anni Moisala Date: Wed, 12 Aug 2026 10:37:19 +0300 Subject: [PATCH 82/82] Update README.md --- AWQ/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AWQ/README.md b/AWQ/README.md index 5b5a068..fa27dbf 100644 --- a/AWQ/README.md +++ b/AWQ/README.md @@ -10,7 +10,7 @@ In order to target weight and activation scaling locations within the model, the To run AWQ quantization scripts on LUMI, you can use the `gptqmodel` library that already ships inside the LUMI AI Factory container — no extra packages or virtual environment are required for AWQ. `torch` , `transformers`, and `datasets` are also already provided in the container. -Load the `Singularity` container environment and set the container image path. Later, create virtual environment inside the container and install the packages. +The script loads the `Singularity` container environment and sets the container image path: ```bash module purge @@ -26,7 +26,7 @@ export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-2026073 ## Roihu The CSC preinstalled PyTorch module covers most of the libraries needed to run these examples -(torch, transformers, datasets, accelerate). The rest can be installed on top of the module in a virtual environment. +(torch, transformers, datasets, accelerate). Llmcompressor can be installed on top of the module in a virtual environment. ### Load the module ```bash