netron/source/gguf-metadata.json

[
  {
    "name": "llama",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "llama4",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "deci",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen2vl",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen3",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true, "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen3vl",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true, "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "dream",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "llada",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "gemma",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "gemma2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "gemma3",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "gemma3n",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "gemma-embedding",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "falcon",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ],
      "parallel_ffn": true
    }
  },
  {
    "name": "gpt2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "learned", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "gptneox",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ],
      "parallel_ffn": true
    }
  },
  {
    "name": "gptj",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ],
      "parallel_ffn": true
    }
  },
  {
    "name": "bloom",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "alibi", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "starcoder",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "learned", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "starcoder2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "baichuan",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "internlm2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "codeshell",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "orion",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "refact",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "xverse",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "stablelm",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true, "has_qk_norm": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "phi2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ],
      "parallel_ffn": true
    }
  },
  {
    "name": "phi3",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "phimoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "olmo",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "olmo2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "olmoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "exaone",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "exaone4",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "exaone-moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "chameleon",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "chatglm",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "glm4",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "glm4moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "glm-dsa",
    "family": "mla",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_LATENT_ATTENTION", "category": "Attention", "tensors": ["attn_q_a", "attn_q_b", "attn_kv_a_mqa", "attn_kv_b", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "command-r",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "cohere2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "plamo",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "plamo3",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "mpt",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "alibi", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "nemotron",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "openelm",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "maincoder",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "arcee",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "mistral3",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "hunyuan-dense",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "pangu-embedded",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "smollm3",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "dots1",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "ernie4_5",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "apertus",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "granite",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "granitemoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "jais",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "alibi", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "jais2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "seed_oss",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "rnd1",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "cogvlm",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "minimax-m2",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "minicpm",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "step35",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true, "has_gate": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "llama-embed",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "smallthinker",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "mimo2",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "lfm2",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "lfm2moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "gpt-oss",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen2moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen3moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true, "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen3vlmoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_bias": true, "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "grok",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "arctic",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "dbrx",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "deepseek",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "ernie4_5-moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "hunyuan-moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "bailingmoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "bailingmoe2",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "afmoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true, "has_gate": true },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "llada-moe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "grovemoe",
    "family": "decoder-moe",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "deepseek2",
    "family": "mla",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_LATENT_ATTENTION", "category": "Attention", "tensors": ["attn_q_a", "attn_q_b", "attn_kv_a_mqa", "attn_kv_b", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "minicpm3",
    "family": "mla",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_LATENT_ATTENTION", "category": "Attention", "tensors": ["attn_q_a", "attn_q_b", "attn_kv_a_mqa", "attn_kv_b", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "plm",
    "family": "mla",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_LATENT_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_kv_a_mqa", "attn_kv_b", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "mamba",
    "family": "ssm",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ssm", "type": "MAMBA", "tensors": ["ssm_in", "ssm_conv1d", "ssm_x", "ssm_dt", "ssm_a", "ssm_d", "ssm_out"] }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "mamba2",
    "family": "ssm",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ssm", "type": "MAMBA2", "tensors": ["ssm_in", "ssm_conv1d", "ssm_dt", "ssm_a", "ssm_d", "ssm_norm", "ssm_out"] }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "rwkv6",
    "family": "rwkv",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "time_mix", "type": "RWKV", "category": "Layer", "tensors": ["time_mix_key", "time_mix_value", "time_mix_receptance", "time_mix_gate", "time_mix_first", "time_mix_decay", "time_mix_lerp_x", "time_mix_lerp_k", "time_mix_lerp_v", "time_mix_lerp_r", "time_mix_lerp_g", "time_mix_lerp_w", "time_mix_w1", "time_mix_w2", "time_mix_decay_w1", "time_mix_decay_w2", "time_mix_ln", "time_mix_output"] },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "channel_mix", "type": "RWKV", "category": "Layer", "tensors": ["channel_mix_key", "channel_mix_receptance", "channel_mix_value", "channel_mix_lerp_k", "channel_mix_lerp_r"] }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "rwkv6qwen2",
    "family": "rwkv",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "time_mix", "type": "RWKV", "category": "Layer", "tensors": ["time_mix_key", "time_mix_value", "time_mix_receptance", "time_mix_gate", "time_mix_first", "time_mix_decay", "time_mix_lerp_x", "time_mix_lerp_k", "time_mix_lerp_v", "time_mix_lerp_r", "time_mix_lerp_g", "time_mix_lerp_w", "time_mix_w1", "time_mix_w2", "time_mix_decay_w1", "time_mix_decay_w2", "time_mix_ln", "time_mix_output"] },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "rwkv7",
    "family": "rwkv",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "time_mix", "type": "RWKV", "category": "Layer", "tensors": ["time_mix_key", "time_mix_value", "time_mix_receptance", "time_mix_lerp_fused", "time_mix_w0", "time_mix_w1", "time_mix_w2", "time_mix_a0", "time_mix_a1", "time_mix_a2", "time_mix_v0", "time_mix_v1", "time_mix_v2", "time_mix_g1", "time_mix_g2", "time_mix_k_k", "time_mix_k_a", "time_mix_r_k", "time_mix_ln", "time_mix_output"] },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "channel_mix", "type": "RWKV", "category": "Layer", "tensors": ["channel_mix_key", "channel_mix_value", "channel_mix_lerp_k"] }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "arwkv7",
    "family": "rwkv",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "time_mix", "type": "RWKV", "category": "Layer", "tensors": ["time_mix_key", "time_mix_value", "time_mix_receptance", "time_mix_lerp_fused", "time_mix_w0", "time_mix_w1", "time_mix_w2", "time_mix_a0", "time_mix_a1", "time_mix_a2", "time_mix_v0", "time_mix_v1", "time_mix_v2", "time_mix_g1", "time_mix_g2", "time_mix_k_k", "time_mix_k_a", "time_mix_r_k", "time_mix_ln", "time_mix_output"] },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "jamba",
    "family": "hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_ssm",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "none" },
          "ssm": { "type": "ssm", "subtype": "mamba", "tensors": ["ssm_in", "ssm_conv1d", "ssm_x", "ssm_dt", "ssm_a", "ssm_d", "ssm_out"] }
        },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "ffn_or_moe",
          "type": "HYBRID_FFN",
          "category": "Layer",
          "dense": { "type": "ffn", "subtype": "swiglu", "tensors": ["ffn_gate", "ffn_up", "ffn_down"] },
          "moe": { "type": "moe", "subtype": "swiglu", "tensors": ["ffn_gate_inp", "ffn_gate_exps", "ffn_up_exps", "ffn_down_exps"] }
        }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "falcon-h1",
    "family": "hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ssm", "type": "MAMBA2", "tensors": ["ssm_in", "ssm_conv1d", "ssm_dt", "ssm_a", "ssm_d", "ssm_norm", "ssm_out"], "parallel": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "granitehybrid",
    "family": "hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_ssm",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
          "ssm": { "type": "ssm", "subtype": "mamba2", "tensors": ["ssm_in", "ssm_conv1d", "ssm_dt", "ssm_a", "ssm_d", "ssm_norm", "ssm_out"] }
        },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "ffn_or_moe",
          "type": "HYBRID_FFN",
          "category": "Layer",
          "dense": { "type": "ffn", "subtype": "swiglu", "tensors": ["ffn_gate", "ffn_up", "ffn_down"] },
          "moe": { "type": "moe", "subtype": "swiglu", "tensors": ["ffn_gate_inp", "ffn_gate_exps", "ffn_up_exps", "ffn_down_exps"], "has_shared_expert": true }
        }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "nemotron_h",
    "family": "hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_ssm",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
          "ssm": { "type": "ssm", "subtype": "mamba2", "tensors": ["ssm_in", "ssm_conv1d", "ssm_dt", "ssm_a", "ssm_d", "ssm_norm", "ssm_out"] }
        },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "nemotron_h_moe",
    "family": "hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_ssm",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
          "ssm": { "type": "ssm", "subtype": "mamba2", "tensors": ["ssm_in", "ssm_conv1d", "ssm_dt", "ssm_a", "ssm_d", "ssm_norm", "ssm_out"] }
        },
        {
          "name": "ffn_or_moe",
          "type": "HYBRID_FFN",
          "category": "Layer",
          "dense": { "type": "ffn", "subtype": "gelu", "tensors": ["ffn_up", "ffn_down"] },
          "moe": { "type": "moe", "subtype": "gelu", "tensors": ["ffn_gate_inp", "ffn_up_exps", "ffn_down_exps"], "has_shared_expert": true }
        }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "plamo2",
    "family": "hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_ssm",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
          "ssm": { "type": "ssm", "subtype": "mamba2", "tensors": ["ssm_in", "ssm_conv1d", "ssm_x", "ssm_dt", "ssm_a", "ssm_d", "ssm_out"] }
        },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "ffn_post_norm", "type": "RMS_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "bert",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "token_types", "type": "EMBEDDING", "category": "Transform" },
        { "name": "position_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "learned", "has_bias": true },
        { "name": "attn_output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "layer_output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "nomic-bert",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "token_types", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "attn_output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "layer_output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "nomic-bert-moe",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "token_types", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "attn_output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" },
        { "name": "layer_output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "jina-bert-v2",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "token_types", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "alibi", "has_qk_norm": true },
        { "name": "attn_output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "layer_output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "output":       []
    }
  },
  {
    "name": "jina-bert-v3",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "token_types", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "learned", "has_bias": true },
        { "name": "attn_output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" },
        { "name": "layer_output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "modern-bert",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "neo-bert",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "paddleocr",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "t5",
    "family": "encoder-decoder",
    "graph": {
      "encoder": {
        "input": [
          { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
        ],
        "blocks": [
          { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
          { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["enc.attn_q", "enc.attn_k", "enc.attn_v", "enc.attn_o"], "position_encoding": "relative_bias" },
          { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
          { "name": "ffn_up", "type": "MUL_MAT" },
          { "name": "ffn_down", "type": "MUL_MAT" }
        ],
        "output": [
          { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" }
        ]
      },
      "decoder": {
        "input": [
          { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
        ],
        "blocks": [
          { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
          { "name": "self_attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["dec.attn_q", "dec.attn_k", "dec.attn_v", "dec.attn_o"], "position_encoding": "relative_bias" },
          { "name": "cross_attn_norm", "type": "RMS_NORM", "category": "Normalization" },
          { "name": "cross_attention", "type": "CROSS_ATTENTION", "category": "Attention", "tensors": ["dec.cross_attn_q", "dec.cross_attn_k", "dec.cross_attn_v", "dec.cross_attn_o"] },
          { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
          { "name": "ffn_up", "type": "MUL_MAT" },
          { "name": "ffn_down", "type": "MUL_MAT" }
        ],
        "output": [
          { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
          { "name": "output", "type": "MUL_MAT" }
        ]
      }
    }
  },
  {
    "name": "t5encoder",
    "family": "encoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["enc.attn_q", "enc.attn_k", "enc.attn_v", "enc.attn_o"], "position_encoding": "relative_bias" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen3next",
    "family": "delta-hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_delta",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
          "delta_net": { "type": "delta_net", "tensors": ["ssm_in", "ssm_conv1d", "ssm_dt", "ssm_a", "ssm_norm", "ssm_ba", "ssm_out"] }
        },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen35",
    "family": "delta-hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_delta",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
          "delta_net": { "type": "delta_net", "tensors": ["ssm_a", "ssm_conv1d", "ssm_dt", "ssm_norm", "ssm_beta", "ssm_alpha", "ssm_out"] }
        },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen35moe",
    "family": "delta-hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_delta",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mha", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_qk_norm": true },
          "delta_net": { "type": "delta_net", "tensors": ["ssm_a", "ssm_conv1d", "ssm_dt", "ssm_norm", "ssm_beta", "ssm_alpha", "ssm_out"] }
        },
        { "name": "attn_post_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "kimi-linear",
    "family": "delta-hybrid",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        {
          "name": "attention_or_delta",
          "type": "HYBRID",
          "category": "Layer",
          "attention": { "type": "attention", "subtype": "mla", "tensors": ["attn_q_a", "attn_q_b", "attn_kv_a_mqa", "attn_kv_b", "attn_output"], "position_encoding": "rope" },
          "delta_net": { "type": "delta_net", "subtype": "kda", "tensors": ["ssm_conv1d_q", "ssm_conv1d_k", "ssm_conv1d_v", "ssm_f_a", "ssm_f_b", "ssm_beta", "ssm_a", "ssm_g_a", "ssm_g_b", "ssm_dt", "ssm_norm"] }
        },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate_inp", "type": "MUL_MAT" },
        { "name": "ffn_gate_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_up_exps", "type": "MUL_MAT_ID" },
        { "name": "ffn_down_exps", "type": "MUL_MAT_ID" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "bitnet",
    "family": "bitnet",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_q", "attn_k", "attn_v", "attn_output"], "position_encoding": "rope", "has_sub_norm": true },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "clip",
    "family": "vision-encoder",
    "graph": {
      "input": [
        { "name": "v.patch_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "v.position_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "v.class_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "input_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["v.attn_qkv", "v.attn_out"], "position_encoding": "learned" },
        { "name": "post_attn_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "v.post_ln", "type": "LAYER_NORM", "category": "Normalization" }
      ]
    }
  },
  {
    "name": "wavtokenizer-dec",
    "family": "audio-decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" },
        { "name": "token_embd_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "conv1d", "type": "CONV_1D", "category": "Layer" }
      ],
      "blocks": [
        { "name": "convnext", "type": "CONV_NEXT", "category": "Layer", "tensors": ["convnext.dw", "convnext.norm", "convnext.pw1", "convnext.pw2", "convnext.gamma"] },
        { "name": "posnet", "type": "POS_NET", "category": "Layer", "tensors": ["posnet.conv1", "posnet.conv2", "posnet.norm", "posnet.norm1", "posnet.norm2", "posnet.attn_norm", "posnet.attn_q", "posnet.attn_k", "posnet.attn_v", "posnet.attn_output"] }
      ],
      "output": [
        { "name": "output_norm", "type": "LAYER_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  },
  {
    "name": "qwen",
    "family": "decoder",
    "graph": {
      "input": [
        { "name": "token_embd", "type": "EMBEDDING", "category": "Transform" }
      ],
      "blocks": [
        { "name": "attn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "attention", "type": "MULTI_HEAD_ATTENTION", "category": "Attention", "tensors": ["attn_qkv", "attn_output"], "position_encoding": "rope" },
        { "name": "ffn_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "ffn_gate", "type": "MUL_MAT" },
        { "name": "ffn_up", "type": "MUL_MAT" },
        { "name": "ffn_down", "type": "MUL_MAT" }
      ],
      "output": [
        { "name": "output_norm", "type": "RMS_NORM", "category": "Normalization" },
        { "name": "output", "type": "MUL_MAT" }
      ]
    }
  }
]