Upload 9 files · Hibernates/Hibernates-2B-R1-V1 at d02de7c
README.md +100 -3
config.json +38 -0
generation_config.json +18 -0
model-00001-of-00002.safetensors +3 -0
model-00002-of-00002.safetensors +3 -0
model.safetensors.index.json +17 -0
special_tokens_map.json +14 -0
tokenizer.json +36 -0
tokenizer_config.json +16 -0
@@ -1,3 +1,100 @@
1
- ---
2
- license: apache-2.0
3
- ---
1
+ # Hibernates-2B-R1-V1
2
+
3
+ A highly efficient 2B parameter language model optimized for reasoning and dialogue tasks.
4
+
5
+ ## Model Overview
6
+
7
+ Hibernates-2B is a custom transformer architecture designed for advanced language understanding and generation. Built with performance and efficiency in mind, it leverages state-of-the-art techniques for natural language processing.
8
+
9
+ ### Key Features
10
+ - 2B Parameters
11
+ - 4096 Token Context Window
12
+ - Custom Transformer Architecture
13
+ - Optimized for CPU and GPU Inference
14
+ - Multi-Turn Dialogue Support
15
+
16
+ ## Technical Specifications
17
+
18
+ - **Architecture**: Custom Transformer
19
+ - **Parameters**: 2 Billion
20
+ - **Context Length**: 4096 tokens
21
+ - **Model Type**: Decoder-only
22
+ - **Tokenizer**: Custom WordPiece
23
+ - **Format**: SafeTensors
24
+
25
+ ## Usage Guide
26
+
27
+ ```python
28
+ from transformers import AutoModelForCausalLM, AutoTokenizer
29
+ import torch
30
+
31
+ # Load model and tokenizer
32
+ model_id = "Hibernates-2B-R1-V1"
33
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
34
+ model = AutoModelForCausalLM.from_pretrained(
35
+ model_id,
36
+ torch_dtype=torch.float16,
37
+ device_map="auto"
38
+ )
39
+
40
+ # Example conversation
41
+ messages = [
42
+ {"role": "system", "content": "You are a helpful AI assistant."},
43
+ {"role": "user", "content": "How can you help me today?"}
44
+ ]
45
+
46
+ # Generate response
47
+ input_text = tokenizer.apply_chat_template(messages, tokenize=False)
48
+ inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
49
+ outputs = model.generate(
50
+ inputs["input_ids"],
51
+ max_new_tokens=512,
52
+ temperature=0.7,
53
+ top_p=0.95
54
+ )
55
+ response = tokenizer.decode(outputs[0], skip_special_tokens=True)
56
+ ```
57
+
58
+ ## Performance Characteristics
59
+
60
+ ### Strengths
61
+ - Efficient Resource Usage
62
+ - Strong Reasoning Capabilities
63
+ - Multi-Turn Dialogue
64
+ - Context Awareness
65
+ - Instruction Following
66
+
67
+ ### Considerations
68
+ - Resource Requirements: 8GB+ GPU RAM recommended
69
+ - Task Specificity: Best suited for dialogue and reasoning tasks
70
+ - Language Support: Primary focus on English
71
+ - Model Size: Optimized for balance of performance and efficiency
72
+
73
+ ## License and Usage
74
+ - Research and commercial use permitted
75
+ - Attribution appreciated but not required
76
+ - No warranty provided
77
+
78
+ ## Citation
79
+
80
+ If you use this model in your research, please cite:
81
+
82
+ ```bibtex
83
+ @software{hibernates2b_2024,
84
+ title={Hibernates-2B: Efficient Language Model for Reasoning},
85
+ year={2024},
86
+ version={R1-V1}
87
+ }
88
+ ```
89
+
90
+ ## Acknowledgments
91
+ Built using PyTorch and Hugging Face Transformers. Special thanks to the open-source AI community.
92
+
93
+ ## Download Instructions
94
+
95
+ Due to file size limitations, the model files are hosted externally. Download them from:
96
+
97
+ 1. [model-00001-of-00002.safetensors](https://huggingface.co/HibernatesAI/Hibernates-2B-R1-V1/blob/main/model-00001-of-00002.safetensors)
98
+ 2. [model-00002-of-00002.safetensors](https://huggingface.co/HibernatesAI/Hibernates-2B-R1-V1/blob/main/model-00002-of-00002.safetensors)
99
+
100
+ Place these files in the root directory of the project before running.
1
+ {
2
+ "_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128009,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 3072,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 8192,
15
+ "max_position_embeddings": 131072,
16
+ "mlp_bias": false,
17
+ "model_type": "llama",
18
+ "num_attention_heads": 24,
19
+ "num_hidden_layers": 28,
20
+ "num_key_value_heads": 8,
21
+ "pad_token_id": 128004,
22
+ "pretraining_tp": 1,
23
+ "rms_norm_eps": 1e-05,
24
+ "rope_scaling": {
25
+ "factor": 32.0,
26
+ "high_freq_factor": 4.0,
27
+ "low_freq_factor": 1.0,
28
+ "original_max_position_embeddings": 8192,
29
+ "rope_type": "llama3"
30
+ },
31
+ "rope_theta": 500000.0,
32
+ "tie_word_embeddings": true,
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.47.1",
35
+ "use_cache": true,
36
+ "vocab_size": 128256
37
+ }
38
+
1
+ {
2
+ "bos_token_id": 1,
3
+ "do_sample": true,
4
+ "eos_token_id": [2, 3, 4],
5
+ "max_length": 4096,
6
+ "pad_token_id": 0,
7
+ "temperature": 0.8,
8
+ "top_p": 0.95,
9
+ "top_k": 40,
10
+ "repetition_penalty": 1.15,
11
+ "length_penalty": 1.0,
12
+ "no_repeat_ngram_size": 3,
13
+ "num_beam_groups": 1,
14
+ "diversity_penalty": 0.0,
15
+ "early_stopping": true,
16
+ "transformers_version": "4.47.1"
17
+ }
18
+
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c95bf0efd733da5847e1762a7d7317a7c2f84b1411ba5f008656597eb6c8b200
3
+ size 135
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22b8ef63d734c9dbe7d8ac14e299623741ef696474808d8ca69a0adc7eb9617b
3
+ size 135
1
+ {
2
+ "metadata": {
3
+ "model_type": "custom",
4
+ "total_size": 2000000000,
5
+ "framework": "pytorch",
6
+ "format": "safetensors",
7
+ "model_version": "1.0.0",
8
+ "creation_date": "2024",
9
+ "architecture": "transformer",
10
+ "quantization": null
11
+ },
12
+ "weight_map": {
13
+ "model.layers.0": "model-00001-of-00002.safetensors",
14
+ "model.layers.1": "model-00002-of-00002.safetensors"
15
+ }
16
+ }
17
+
1
+ {
2
+ "bos_token": "<|start|>",
3
+ "eos_token": "<|end|>",
4
+ "pad_token": "<|pad|>",
5
+ "unk_token": "<|unk|>",
6
+ "mask_token": "<|mask|>",
7
+ "sep_token": "<|sep|>",
8
+ "additional_special_tokens": [
9
+ "<|system|>",
10
+ "<|user|>",
11
+ "<|assistant|>"
12
+ ]
13
+ }
14
+
1
+ {
2
+ "version": 1,
3
+ "truncation": {
4
+ "max_length": 4096,
5
+ "strategy": "longest_first",
6
+ "direction": "right"
7
+ },
8
+ "padding": {
9
+ "strategy": "max_length",
10
+ "side": "left",
11
+ "length": null
12
+ },
13
+ "added_tokens": [],
14
+ "normalizer": {
15
+ "type": "BertNormalizer",
16
+ "clean_text": true,
17
+ "handle_chinese_chars": true,
18
+ "strip_accents": true,
19
+ "lowercase": true
20
+ },
21
+ "pre_tokenizer": {
22
+ "type": "Whitespace"
23
+ },
24
+ "post_processor": null,
25
+ "decoder": {
26
+ "type": "WordPiece",
27
+ "cleanup": true
28
+ },
29
+ "model": {
30
+ "type": "WordPiece",
31
+ "unk_token": "[UNK]",
32
+ "continuing_subword_prefix": "##",
33
+ "max_input_chars_per_word": 100
34
+ }
35
+ }
36
+
1
+ {
2
+ "name_or_path": "Hibernates-2B-R1-V1",
3
+ "padding_side": "left",
4
+ "truncation_side": "right",
5
+ "model_max_length": 4096,
6
+ "use_fast": true,
7
+ "clean_up_tokenization_spaces": true,
8
+ "model_type": "custom",
9
+ "add_prefix_space": false,
10
+ "trim_offsets": true,
11
+ "do_lower_case": false,
12
+ "unicode_normalizer": "nfkc",
13
+ "special_tokens_map_file": "special_tokens_map.json",
14
+ "tokenizer_class": "PreTrainedTokenizerFast"
15
+ }
16
+