The Complete Qwen 3.6 Guide

# 1. Build llama.cpp with CUDA support
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
cmake -B build -DGGML_CUDA=ON
cmake --build build --config Release

# For Apple Silicon (Metal)
cmake -B build -DGGML_METAL=ON
cmake --build build --config Release

# 2. Download GGUF model from Hugging Face
# Q4_K_M — best size/quality tradeoff (~20 GB)
wget https://huggingface.co/Qwen/Qwen3.6-35B-A3B-GGUF/resolve/main/qwen3.6-35b-a3b-q4_k_m.gguf

# 3. Run inference
./build/bin/llama-cli \
  -m qwen3.6-35b-a3b-q4_k_m.gguf \
  -n 512 \
  --n-gpu-layers 35 \
  -p "You are a helpful assistant.\n\nUser: Explain quantum computing\nAssistant:"

# 4. Run as local server (OpenAI-compatible)
./build/bin/llama-server \
  -m qwen3.6-35b-a3b-q4_k_m.gguf \
  --n-gpu-layers 35 \
  --host 0.0.0.0 \
  --port 8080 \
  --ctx-size 8192

# 5. CPU-only (slower but works on any machine)
./build/bin/llama-cli \
  -m qwen3.6-35b-a3b-q2_k.gguf \
  --n-gpu-layers 0 \
  --threads 16

# 1. Install vLLM (requires CUDA 12.1+, Python 3.9+)
pip install vllm

# 2. Launch Qwen 3.6 with vLLM server
python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen3.6-35B-A3B \
  --tensor-parallel-size 1 \
  --max-model-len 32768 \
  --port 8000

# Multi-GPU (2x GPU for FP16 full precision)
python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen3.6-35B-A3B \
  --tensor-parallel-size 2 \
  --dtype bfloat16

# 3. Query the API
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen3.6-35B-A3B",
    "messages": [{"role": "user", "content": "Write a Python fibonacci function"}],
    "max_tokens": 512
  }'

# 4. Use with OpenAI SDK
pip install openai
from openai import OpenAI
client = OpenAI(base_url="http://localhost:8000/v1", api_key="none")
response = client.chat.completions.create(
    model="Qwen/Qwen3.6-35B-A3B",
    messages=[{"role": "user", "content": "Hello!"}]
)

# 1. Install dependencies
pip install transformers torch accelerate

# 2. Basic inference
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

model_name = "Qwen/Qwen3.6-35B-A3B"

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Explain the MoE architecture"}
]

text = tokenizer.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer([text], return_tensors="pt").to(model.device)

with torch.no_grad():
    outputs = model.generate(**inputs, max_new_tokens=512)

print(tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:]))

# 3. With 4-bit quantization (saves VRAM)
from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16
)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto"
)