{
  "_published": {
    "source": "docs/measurements/gguf-quantize-run-2026-11-15.json",
    "published": "2026-11-15",
    "publisher": "J4SGON S.L., vorluxai.com",
    "index": "https://vorluxai.com/data/",
    "cleaned": "Copied from our internal repository by scripts/publish_measurements.py. Every number is unchanged. Redacted: nothing. Hostnames (spark-43d5, spark-7180) and the PRIME / spark2 labels are kept: the posts use them.",
    "redactions": {}
  },
  "title": "llama-quantize on one 7B model: a real FP16 to Q4_K_M run, dry-run sizes per type, and two refusals",
  "date": "2026-11-15",
  "machine": {
    "alias": "prime",
    "hostname": "spark-43d5",
    "hardware": "NVIDIA DGX Spark, GB10, 128 GB unified memory, 20 CPU cores (aarch64)",
    "kernel": "6.17.0-1032-nvidia",
    "gpu_used": false,
    "note": "llama-quantize ran on the CPU only; no model was loaded on the GPU for this file"
  },
  "tool": {
    "name": "llama-quantize",
    "llama_cpp_commit": "cb295bf59663cd3577389315636772f4060bd1f5",
    "llama_cpp_commit_date": "2026-07-06",
    "build_line": "llama_print_build_info: build = 1 (cb295bf), built with GNU 13.3.0 for Linux aarch64",
    "note": "our local build is from July 2026; upstream master on 2026-11-15 lists two options this build does not have (the Q2_0 type and --max-buffer-size)"
  },
  "input": {
    "model": "qwen2.5:7b-instruct-fp16 from the Ollama library (the GGUF blob Ollama stores on disk, located with `ollama show --modelfile`)",
    "general_name": "Qwen2.5 7B Instruct",
    "tensors": 339,
    "file_bytes": 15237853600,
    "model_size_mib": 14526.27,
    "bits_per_weight": 16.00
  },
  "quantize_run": {
    "started_at": "2026-11-15T09:11:25Z",
    "finished_at": "2026-11-15T09:12:12Z",
    "command": "nice llama-quantize <fp16 blob> qwen2.5-7b-instruct-Q4_K_M.gguf Q4_K_M 8",
    "threads": 8,
    "exit_code": 0,
    "quantize_time_ms": 46515.99,
    "wall_clock_s": 46.52,
    "cpu_user_s": 255.26,
    "max_resident_set_kb": 14885452,
    "mem_available_before_kb": 65467156,
    "mem_available_after_kb": 67705116,
    "output_file_bytes": 4683073952,
    "output_sha256": "b36688f041b3267d739af5b69df342774705b4dcba2394fc9a9ae0b621231c78",
    "reported_model_size_mib": 14526.27,
    "reported_quant_size_mib": 4460.45,
    "reported_bits_per_weight": 4.91,
    "output_metadata": {
      "general.file_type": 15,
      "general.quantization_version": 2,
      "general.basename": "Qwen2.5",
      "general.size_label": "7B",
      "general.finetune": "Instruct"
    },
    "output_tensor_types": {
      "Q4_K": 169,
      "Q6_K": 29,
      "F32": 141
    },
    "q6_k_tensors": "output.weight, plus attn_v.weight and ffn_down.weight in 14 of the 28 layers (0, 1, 2, 5, 8, 11, 14, 17, 20, 23, 24, 25, 26, 27)",
    "f32_tensors": "norm weights and biases, each well under 0.1 MiB",
    "last_lines": [
      "[ 336/ 339] blk.27.ffn_down.weight               - [ 18944,   3584,      1,      1], type =    f16, converting to q6_K .. size =   129.50 MiB ->    53.12 MiB",
      "[ 337/ 339] blk.27.ffn_gate.weight               - [  3584,  18944,      1,      1], type =    f16, converting to q4_K .. size =   129.50 MiB ->    36.42 MiB",
      "[ 338/ 339] blk.27.ffn_norm.weight               - [  3584,      1,      1,      1], type =    f32, size =    0.014 MiB",
      "[ 339/ 339] blk.27.ffn_up.weight                 - [  3584,  18944,      1,      1], type =    f16, converting to q4_K .. size =   129.50 MiB ->    36.42 MiB",
      "llama_model_quantize_impl: model size  = 14526.27 MiB (16.00 BPW)",
      "llama_model_quantize_impl: quant size  =  4460.45 MiB (4.91 BPW)",
      "llama_quantize: quantize time = 46515.99 ms",
      "llama_quantize:    total time = 46515.99 ms"
    ],
    "not_done": "the output file was read back with gguf-py (metadata and tensor types above) but was not loaded for inference and not imported into Ollama"
  },
  "dry_run_sizes": {
    "command": "llama-quantize --dry-run <fp16 blob> <TYPE>",
    "note": "sizes as printed by `quant size`, in MiB, with the bits per weight the tool reports; nothing is written. IQ types were sized without an importance matrix",
    "rows": [
      {"type": "Q2_K", "quant_size_mib": 2870.55, "bits_per_weight": 3.16},
      {"type": "IQ2_XS", "quant_size_mib": 2348.96, "bits_per_weight": 2.59, "warning": "llama_model_quantize_impl: WARNING: dry run completed successfully, but actually completing this quantization will require an imatrix!"},
      {"type": "IQ3_M", "quant_size_mib": 3402.77, "bits_per_weight": 3.75},
      {"type": "Q3_K_M", "quant_size_mib": 3626.29, "bits_per_weight": 3.99},
      {"type": "IQ4_XS", "quant_size_mib": 4047.72, "bits_per_weight": 4.46},
      {"type": "Q4_0", "quant_size_mib": 4220.43, "bits_per_weight": 4.65},
      {"type": "Q4_K_S", "quant_size_mib": 4245.58, "bits_per_weight": 4.68},
      {"type": "Q4_K_M", "quant_size_mib": 4460.45, "bits_per_weight": 4.91},
      {"type": "Q5_K_S", "quant_size_mib": 5063.27, "bits_per_weight": 5.58},
      {"type": "Q5_K_M", "quant_size_mib": 5186.92, "bits_per_weight": 5.71},
      {"type": "Q6_K", "quant_size_mib": 5958.79, "bits_per_weight": 6.56},
      {"type": "Q8_0", "quant_size_mib": 7717.68, "bits_per_weight": 8.50}
    ]
  },
  "refusals": [
    {
      "case": "quantising a file that is already quantised (the llama3.1:8b Q4_K_M blob, 4920738944 bytes) to Q3_K_M without --allow-requantize",
      "command": "llama-quantize <q4_k_m blob> out.gguf Q3_K_M 4",
      "exit_code": 1,
      "stderr": [
        "llama_model_quantize: failed to quantize: requantizing from type q4_K is disabled",
        "llama_quantize: failed to quantize model from '<q4_k_m blob>'"
      ],
      "note": "the same command with --dry-run does not raise this error; it prints a size (3825.27 MiB, 4.00 BPW) and exits 0"
    },
    {
      "case": "quantising the FP16 file to IQ2_XS without an importance matrix",
      "command": "llama-quantize <fp16 blob> out.gguf IQ2_XS 4",
      "exit_code": 1,
      "stderr": [
        "ERROR: this quantization requires an importance matrix!",
        "llama_model_quantize: failed to quantize: this quantization requires an imatrix!",
        "llama_quantize: failed to quantize model from '<fp16 blob>'"
      ]
    }
  ],
  "ollama_show": {
    "note": "read-only checks of which quantisation a tag serves, 2026-11-15; no model was loaded",
    "rows": [
      {"host": "spark-43d5", "ollama_version": "0.30.10", "tag": "llama3.1:8b", "parameters": "8.0B", "context_length": 131072, "quantization": "Q4_K_M", "baked_num_ctx": 32768},
      {"host": "spark-43d5", "ollama_version": "0.30.10", "tag": "llava:7b", "parameters": "7B", "context_length": 32768, "quantization": "Q4_0"},
      {"host": "spark-43d5", "ollama_version": "0.30.10", "tag": "qwen2.5:7b-instruct-fp16", "parameters": "7.6B", "quantization": "F16"},
      {"host": "spark-7180", "ollama_version": "0.34.0", "tag": "mistral-small:24b", "parameters": "23.6B", "context_length": 32768, "quantization": "Q4_K_M", "baked_num_ctx": 16384}
    ],
    "create_help_0_30_10": "-q, --quantize string         Quantize model to this level (e.g. q4_K_M)"
  }
}
