Hugging Face's logo

dineshkvr
/
gemma3-270m-tinystories-gpt2-tokenizer

Text Generation
PyTorch
English
gemma3
causal-lm
tinystories
small-language-model
custom_code
Model card Files
xet
Community
1
dineshkvr commited on
Commit
4c797aa
·
verified ·
1 Parent(s): aae6a5b

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +28 -34
config.json CHANGED
@@ -8,43 +8,37 @@
8
  },
9
  "vocab_size": 50257,
10
  "context_length": 32768,
11
- "emb_dim": 1024,
12
- "n_heads": 16,
13
- "n_layers": 24,
14
- "hidden_dim": 4096,
15
- "head_dim": 64,
16
  "qk_norm": true,
17
- "n_kv_groups": 4,
18
- "rope_local_base": 10000,
19
- "rope_base": 10000,
20
- "sliding_window": 128,
21
  "layer_types": [
22
- "transformer",
23
- "transformer",
24
- "transformer",
25
- "transformer",
26
- "transformer",
27
- "transformer",
28
- "transformer",
29
- "transformer",
30
- "transformer",
31
- "transformer",
32
- "transformer",
33
- "transformer",
34
- "transformer",
35
- "transformer",
36
- "transformer",
37
- "transformer",
38
- "transformer",
39
- "transformer",
40
- "transformer",
41
- "transformer",
42
- "transformer",
43
- "transformer",
44
- "transformer",
45
- "transformer"
46
  ],
47
- "query_pre_attn_scalar": 1.0,
48
  "model_type": "gemma3",
49
  "torch_dtype": "bfloat16"
50
  }
 
8
  },
9
  "vocab_size": 50257,
10
  "context_length": 32768,
11
+ "emb_dim": 640,
12
+ "n_heads": 4,
13
+ "n_layers": 18,
14
+ "hidden_dim": 2048,
15
+ "head_dim": 256,
16
  "qk_norm": true,
17
+ "n_kv_groups": 1,
18
+ "rope_local_base": 10000.0,
19
+ "rope_base": 1000000.0,
20
+ "sliding_window": 512,
21
  "layer_types": [
22
+ "sliding_attention",
23
+ "sliding_attention",
24
+ "sliding_attention",
25
+ "sliding_attention",
26
+ "sliding_attention",
27
+ "full_attention",
28
+ "sliding_attention",
29
+ "sliding_attention",
30
+ "sliding_attention",
31
+ "sliding_attention",
32
+ "sliding_attention",
33
+ "full_attention",
34
+ "sliding_attention",
35
+ "sliding_attention",
36
+ "sliding_attention",
37
+ "sliding_attention",
38
+ "sliding_attention",
39
+ "full_attention"
 
 
 
 
 
 
40
  ],
41
+ "query_pre_attn_scalar": 256,
42
  "model_type": "gemma3",
43
  "torch_dtype": "bfloat16"
44
  }