Upload 15 files

gpt-2/dataloader.py

@@ -0,0 +1,74 @@
+import os
+import multiprocessing as mp
+import numpy as np
+import tiktoken
+from datasets import load_dataset # pip install datasets
+from tqdm import tqdm # pip install tqdm
+
+# ------------------------------------------
+local_dir = "edu_fineweb10B"
+remote_name = "sample-10BT"
+shard_size = int(1e8) # 100M tokens per shard, total of 100 shards
+
+# create the cache the local directory if it doesn't exist yet
+DATA_CACHE_DIR = os.path.join(os.path.dirname(__file__), local_dir)
+os.makedirs(DATA_CACHE_DIR, exist_ok=True)
+
+# download the dataset
+fw = load_dataset("HuggingFaceFW/fineweb-edu", name=remote_name, split="train")
+
+# init the tokenizer
+enc = tiktoken.get_encoding("gpt2")
+eot = enc._special_tokens['<|endoftext|>'] # end of text token
+def tokenize(doc):
+    # tokenizes a single document and returns a numpy array of uint16 tokens
+    tokens = [eot] # the special token delimits all documents
+    tokens.extend(enc.encode_ordinary(doc["text"]))
+    tokens_np = np.array(tokens)
+    assert (0 <= tokens_np).all() and (tokens_np < 2**16).all(), "token dictionary too large for uint16"
+    tokens_np_uint16 = tokens_np.astype(np.uint16)
+    return tokens_np_uint16
+
+def write_datafile(filename, tokens_np):
+    np.save(filename, tokens_np)
+
+if __name__ == '__main__':
+    # tokenize all documents and write output shards, each of shard_size tokens (last shard has remainder)
+    nprocs = max(1, os.cpu_count()//2)
+    with mp.Pool(nprocs) as pool:
+        shard_index = 0
+        # preallocate buffer to hold current shard
+        all_tokens_np = np.empty((shard_size,), dtype=np.uint16)
+        token_count = 0
+        progress_bar = None
+        for tokens in pool.imap(tokenize, fw, chunksize=16):
+
+            # is there enough space in the current shard for the new tokens?
+            if token_count + len(tokens) < shard_size:
+                # simply append tokens to current shard
+                all_tokens_np[token_count:token_count+len(tokens)] = tokens
+                token_count += len(tokens)
+                # update progress bar
+                if progress_bar is None:
+                    progress_bar = tqdm(total=shard_size, unit="tokens", desc=f"Shard {shard_index}")
+                progress_bar.update(len(tokens))
+            else:
+                # write the current shard and start a new one
+                split = "val" if shard_index == 0 else "train"
+                filename = os.path.join(DATA_CACHE_DIR, f"edufineweb_{split}_{shard_index:06d}")
+                # split the document into whatever fits in this shard; the remainder goes to next one
+                remainder = shard_size - token_count
+                progress_bar.update(remainder)
+                all_tokens_np[token_count:token_count+remainder] = tokens[:remainder]
+                write_datafile(filename, all_tokens_np)
+                shard_index += 1
+                progress_bar = None
+                # populate the next shard with the leftovers of the current doc
+                all_tokens_np[0:len(tokens)-remainder] = tokens[remainder:]
+                token_count = len(tokens)-remainder
+
+        # write any remaining tokens as the last shard
+        if token_count != 0:
+            split = "val" if shard_index == 0 else "train"
+            filename = os.path.join(DATA_CACHE_DIR, f"edufineweb_{split}_{shard_index:06d}")
+            write_datafile(filename, all_tokens_np[:token_count])

gpt-2/gpt2_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d663ee22770b02eb68070f343ebc621f91493e3e8b146e68eca76cbe919a3114
+size 548294034

gpt-2/lossi.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd7fd7be14551deea46f888e47512faef42894322aacbb833f366fbc382cb05f
+size 1362

gpt-2/lossi_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b40324e09dcb246105d63bee020a2bf3791c2bc8e06b6374cc550c9c52c907c
+size 73225

gpt-2/model.py

@@ -0,0 +1,131 @@
+# model.py
+
+from dataclasses import dataclass
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import inspect
+@dataclass
+class GPTConfig:
+    vocab_size: int = 50257
+    block_size: int = 1024
+    n_layer: int = 12
+    n_head: int = 12
+    n_embd: int = 768 # = 64 * 12
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
+        self.c_proj.NANOGPT_SCALE_INIT = 1
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+        self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size))
+                                     .view(1, 1, config.block_size, config.block_size))
+
+    def forward(self, x):
+        B, T, C = x.size()
+        qkv = self.c_attn(x)
+        q, k, v = qkv.split(self.n_embd, dim=2)
+        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        y = y.transpose(1, 2).contiguous().view(B, T, C)
+        y = self.c_proj(y)
+        return y
+
+class MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.c_fc = nn.Linear(config.n_embd, config.n_embd * 4)
+        self.c_proj = nn.Linear(config.n_embd * 4, config.n_embd)
+        self.gelu = nn.GELU()
+        self.NANOGPT_SCALE_INIT = 1
+        
+    def forward(self, x):
+        x = self.gelu(self.c_fc(x))
+        x = self.c_proj(x)
+        return x
+
+class Block(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.n_embd)
+        self.ln_2 = nn.LayerNorm(config.n_embd)
+        self.attn = CausalSelfAttention(config)
+        self.mlp = MLP(config)
+        
+    def forward(self, x):
+        x = x + self.attn(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+class GPT(nn.Module):
+    def __init__(self, config, master_process):
+        super().__init__()
+        self.master_process = master_process
+        self.config = config
+        self.transformer = nn.ModuleDict(dict(
+            wte = nn.Embedding(config.vocab_size, config.n_embd),
+            wpe = nn.Embedding(config.block_size, config.n_embd),
+            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
+            ln_f = nn.LayerNorm(config.n_embd)            
+        ))
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        self.transformer.wte.weight = self.lm_head.weight
+        self.apply(self._init_weights)
+        if self.master_process:
+            print(f"Model initialized. Model has {sum(p.numel() for p in self.parameters() if p.requires_grad):,} trainable parameters")
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            std = 0.2
+            if hasattr(module, 'NANOGPT_SCALE_INIT'):
+                std*= (2 * self.config.n_layer)**-0.5
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)            
+
+    def forward(self, idx, targets=None):
+        B, T = idx.size()
+        assert T <= self.config.block_size, "Cannot forward, model block size is exhausted." 
+        pos = torch.arange(0, T, dtype=torch.long, device=idx.device)
+        pos_emb = self.transformer.wpe(pos)
+        tok_emb = self.transformer.wte(idx)
+        x = tok_emb + pos_emb
+        for block in self.transformer.h:
+            x = block(x)
+        x = self.transformer.ln_f(x)
+        logits = self.lm_head(x)
+        loss = None
+        if targets is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1)) 
+        return logits, loss
+    
+    def configure_optimizers(self, weight_decay, learning_rate, device):
+        param_dict = {pn: p for pn, p in self.named_parameters()}
+        param_dict = {pn: p for pn, p in param_dict.items() if p.requires_grad}
+
+        decay_params = [p for n, p in param_dict.items() if p.dim() >=2]
+        nodecay_params = [p for n, p in param_dict.items() if p.dim() < 2]
+        optim_groups = [
+            {"params": decay_params, "weight_decay": weight_decay},
+            {"params": nodecay_params, "weight_decay": 0.0},
+        ]
+        num_decay_params = sum(p.numel() for p in decay_params)
+        num_nodecay_params = sum(p.numel() for p in nodecay_params)
+        if self.master_process:
+            print(f"Number of decay parameters tensors: {len(decay_params)}, Number of decay parameters: {num_decay_params:,}")
+            print(f"Number of no decay parameters tensors: {len(nodecay_params)}, Number of no decay parameters: {num_nodecay_params:,}")
+
+        fused_available = 'fused' in inspect.signature(torch.optim.AdamW).parameters
+        use_fused = fused_available and 'cuda' == device
+        if self.master_process:
+            print(f'Using {"fused" if use_fused else "unfused"} AdamW')
+        optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=(0.9, 0.95), eps=1e-8)
+        return optimizer

gpt-2/training_full_dataset.py

@@ -0,0 +1,362 @@
+from dataclasses import dataclass
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from model import GPT, GPTConfig
+import tiktoken
+from torch.utils.data import Dataset, DataLoader, DistributedSampler
+import math
+import matplotlib.pyplot as plt
+from torch.distributed import init_process_group, destroy_process_group
+from torch.nn.parallel import DistributedDataParallel as DDP
+import torch.distributed as dist
+import os
+import signal
+import sys
+import numpy as np
+import time 
+import logging 
+
+def seconds_to_hms(seconds):
+    return time.strftime('%H:%M:%S', time.gmtime(seconds))
+
+
+def signal_handler(sig, frame):
+    print('Gracefully stopping the training process')
+    destroy_process_group()
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+manual_seed = 1339
+torch.manual_seed(manual_seed)
+if torch.cuda.is_available():
+    torch.cuda.manual_seed(manual_seed)
+
+# ***************************#
+# Device Configuration
+# ***************************#
+device = torch.device("cpu")
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+    device = torch.device("mps")
+
+print("Using device:", device)
+
+# ***************************#
+# Tokenizer Setup
+# ***************************#
+enc = tiktoken.get_encoding('gpt2')
+
+
+lossi = []
+val_lossi = []
+
+# ***************************#
+# Load Text Data
+# ***************************#
+with open("tinyshakespeare.txt", "r") as f:
+    text = f.read()
+tokens = enc.encode(text)
+print(f"Number of tokens: {len(tokens):,}")
+# ***************************#
+# Set up DDP
+# ***************************#
+# torchrun command sets the env variables RANK, LOCAL_RANK, and WORLD_SIZE
+ddp = int(os.environ.get('RANK', -1)) != -1  # is this a ddp run?
+if ddp:
+    # use of DDP atm demands CUDA, we set the device appropriately according to rank
+    assert torch.cuda.is_available(), "for now i think we need CUDA for DDP"
+    init_process_group(backend='nccl')
+    ddp_rank = int(os.environ['RANK'])
+    ddp_local_rank = int(os.environ['LOCAL_RANK'])
+    ddp_world_size = int(os.environ['WORLD_SIZE'])
+    device = f'cuda:{ddp_local_rank}'
+    torch.cuda.set_device(device)
+    # this process will do logging, checkpointing etc.
+    master_process = ddp_rank == 0
+else:
+    # vanilla, non-DDP run
+    ddp_rank = 0
+    ddp_local_rank = 0
+    ddp_world_size = 1
+    master_process = True
+
+if master_process:
+    print(f"ddp: {ddp}, rank: {ddp_rank}, local_rank: {ddp_local_rank}, world_size: {ddp_world_size}, master_process: {master_process}")
+    
+# ***************************#
+# Model Configuration
+# ***************************#
+
+gpt = GPT(GPTConfig(vocab_size=50304), master_process).to(device)
+if device == torch.device("cuda"):
+    gpt.compile()
+if ddp:
+    gpt = DDP(gpt, device_ids=[ddp_local_rank])
+
+raw_gpt = gpt.module if ddp else gpt
+
+# ***************************#
+# Dataset and Dataloader
+# ***************************#
+
+def load_tokens(filename):
+    npt = np.load(filename)
+    npt = npt.astype(np.int32) # added after video
+    ptt = torch.tensor(npt, dtype=torch.long)
+    return ptt
+
+class DataLoader_Custom:
+    def __init__(self, B, T, process_rank, num_processes, split, shuffle=False):
+        self.B = B
+        self.T = T
+        self.process_rank = process_rank
+        self.num_processes = num_processes
+        self.shuffle = shuffle
+        assert split in ["train", "val"]
+
+        data_root = "edu_fineweb10B"
+        shards = os.listdir(data_root)
+        shards = [s for s in shards if split in s]
+        shards = sorted(shards)
+        shards = [os.path.join(data_root, s) for s in shards]
+        self.shards = shards
+        assert len(shards) > 0, "No shards found for split {}".format(split)
+        if master_process:
+            print("Found {} shards for split {}".format(len(shards), split))
+        self.current_shard = 0
+        self.tokens = load_tokens(self.shards[self.current_shard])
+        self.current_position = self.B * self.T * self.process_rank
+
+    def next_batch(self):
+        B, T = self.B, self.T
+        buf = self.tokens[self.current_position:self.current_position + B*T+1]
+        x = buf[:-1].view(B, T)
+        y = buf[1:].view(B, T)
+        self.current_position += B*T * self.num_processes
+        if self.current_position + (B*T*self.num_processes+1) > len(self.tokens):
+            self.current_shard = self.current_shard + 1 % len(self.shards)
+            self.tokens = load_tokens(self.shards[self.current_shard])
+            self.current_position = self.B * self.T * self.process_rank
+
+        return x, y
+    
+    def reset(self):
+        self.current_shard = 0
+        self.tokens = load_tokens(self.shards[self.current_shard])
+        self.current_position = self.B * self.T * self.process_rank
+        
+T = 4
+batch_size = 1
+total_batch_size = 2**2  # 524,288 = 2**19, in number of tokens
+assert total_batch_size % (T*batch_size*ddp_world_size) == 0, "Batch size is not divisible by B*T"
+grad_accum_steps = total_batch_size // (T*batch_size*ddp_world_size)
+
+if master_process:
+    print("Total desired batch size: {:,}".format(total_batch_size))
+    print("gradient accumulation steps: {:,}".format(grad_accum_steps))
+
+train_dataloader = DataLoader_Custom(batch_size, T, ddp_local_rank, ddp_world_size, "train")
+val_dataloader = DataLoader_Custom(batch_size, T, ddp_local_rank, ddp_world_size, "val")
+
+
+# ***************************#
+# Text Generation Function
+# ***************************#
+
+
+def generate_text(seed_text, model, enc, max_len=100, print_while_generating=True):
+    if print_while_generating:
+        print(seed_text, end="")
+    model.eval()
+    with torch.no_grad():
+        tokens = enc.encode(seed_text)
+        for _ in range(max_len):
+            x = torch.tensor(tokens[-T:], dtype=torch.long,
+                             device=device).unsqueeze(0)
+            logits, _ = model(x)
+            next_token = torch.argmax(logits[:, -1, :])
+            tokens.append(int(next_token))
+
+            if print_while_generating:
+                print(enc.decode([int(next_token)]), end="")
+        print()
+
+    return enc.decode(tokens)
+
+
+# ***************************#
+# Optimizer Configuration
+# ***************************#
+if ddp:
+    optimizer = raw_gpt.configure_optimizers(
+        weight_decay=0.1, learning_rate=6e-4, device=device)
+else:
+    optimizer = gpt.configure_optimizers(
+        weight_decay=0.1, learning_rate=6e-4, device=device)
+torch.set_float32_matmul_precision('high')
+# ***************************#
+# Learning Rate Scheduler
+# ***************************#
+max_lr = 6e-4
+min_lr = max_lr * 0.1
+warmup_steps = 715
+max_steps = 50
+
+
+def get_lr(step):
+    if step < warmup_steps:
+        return max_lr * (step+1) / warmup_steps
+    if step > max_steps:
+        return min_lr
+    decay_ratio = (step - warmup_steps) / (max_steps - warmup_steps)
+    assert 0 <= decay_ratio <= 1
+    coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))
+    return min_lr + coeff * (max_lr - min_lr)
+
+
+# Check if the device supports bfloat16
+supports_bfloat16 = False
+if device == "cuda":
+    capability = torch.cuda.get_device_capability()
+    if capability[0] >= 8 and capability[1] >= 0:
+        supports_bfloat16 = True
+
+print("Supports bfloat16:", supports_bfloat16)
+
+# ***************************#
+# Training Loop
+# ***************************#
+
+generate_every = 50
+validate_every = 10
+save_every = 5
+t0 = time.time()
+
+# Initialize logging
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+logger = logging.getLogger(__name__)
+
+# Add a file handler
+file_handler = logging.FileHandler('training_log.txt')
+file_handler.setLevel(logging.INFO)
+file_handler.setFormatter(logging.Formatter('%(message)s'))
+logger.addHandler(file_handler)
+
+for step in range(max_steps):
+    
+    loss_accum = 0.0
+    gpt.zero_grad()
+    for minibatchstep in range(grad_accum_steps):
+        x, y = train_dataloader.next_batch()
+        x, y = x.to(device), y.to(device)
+
+        if supports_bfloat16:
+            with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
+                logits, loss = gpt(x, y)
+        else:
+            logits, loss = gpt(x, y)
+        
+        loss = loss / grad_accum_steps
+        loss_accum += loss.detach()
+        if ddp: 
+            gpt.require_backward_grad_sync = (minibatchstep == grad_accum_steps - 1)
+        loss.backward()
+
+    if ddp:
+        dist.all_reduce(loss_accum, op=dist.ReduceOp.AVG)
+    lossi.append(loss_accum.item())
+    norm = torch.nn.utils.clip_grad_norm_(gpt.parameters(), 1.0)
+    lr = get_lr(step)
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr
+    optimizer.step()
+    t_current = time.time()
+    elapsed_time = t_current - t0
+    steps_completed = step + 1
+    avg_time_per_step = elapsed_time / steps_completed
+    remaining_steps = max_steps - steps_completed
+    remaining_time = remaining_steps * avg_time_per_step
+
+    if master_process:
+        logger.info(f'Step {step} | Loss: {loss_accum:.6f} | Norm: {norm:.4f} | LR: {lr:.2e} | Time: {seconds_to_hms(elapsed_time)} | Remaining: {seconds_to_hms(remaining_time)} | Avg Time/Step: {avg_time_per_step:.2f}')
+    
+    if master_process and step % generate_every == 0:    
+        generated_text = generate_text("The king said", gpt, enc, max_len=25, print_while_generating=False)
+        logger.info(f'Generated Text at Step {step}: {generated_text}')
+
+    # Validation step
+    if step % validate_every == 0:
+        if master_process:
+            logger.info("Validating...")
+        gpt.eval()
+        val_loss_accum = 0.0
+        val_dataloader.reset()
+        with torch.no_grad():
+            val_loss_accum
+            val_loss_steps = 20
+            for _ in range(val_loss_steps):
+                x, y = val_dataloader.next_batch()
+                x, y = x.to(device), y.to(device)
+                if supports_bfloat16:
+                    with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
+                        val_logits, val_loss = gpt(x, y)
+                else:
+                    val_logits, val_loss = gpt(x, y)
+                val_loss = val_loss / val_loss_steps
+                val_loss_accum += val_loss.detach()
+        if ddp:
+            dist.all_reduce(val_loss_accum, op=dist.ReduceOp.AVG)
+        if master_process:
+            logger.info(f'Validation Loss: {val_loss_accum}')
+        val_lossi.append(val_loss_accum.item())
+
+    if step % save_every == 0 and master_process:
+        print("Saving model and loss...")
+        torch.save(raw_gpt.state_dict(), "gpt2_step_{}.pth".format(step))
+        torch.save(torch.tensor(lossi), "lossi_step_{}.pth".format(step))
+        torch.save(torch.tensor(val_lossi), "val_lossi_step_{}.pth".format(step))
+        
+# ***************************#
+# Plot Loss
+# ***************************#
+
+plot = True
+if master_process and plot:
+    plt.plot(lossi, label="Train Loss")
+    
+    # Stretch val_lossi to match the length of lossi
+    val_lossi_stretched = np.interp(
+        np.linspace(0, len(val_lossi) - 1, len(lossi)),
+        np.arange(len(val_lossi)),
+        val_lossi
+    )
+    
+    plt.plot(val_lossi_stretched, label="Validation Loss")
+    plt.legend()
+    plt.xlabel("Step")
+    plt.ylabel("Loss")
+
+    plt.show()
+
+# Generate Final Text
+if master_process:
+   print(generate_text("The king said", gpt, enc, max_len=25, print_while_generating=False))
+
+# ***************************#
+# Save Model and Loss
+# ***************************#
+if master_process:
+    torch.save(gpt.state_dict(), "gpt2_shakespeare.pth")
+    torch.save(torch.tensor(lossi), "lossi.pth")
+    torch.save(torch.tensor(val_lossi), "val_lossi.pth")
+
+# ***************************#
+# Cleanup
+# ***************************#
+if ddp:
+    destroy_process_group()
+
+
+import sys; sys.exit(0)

gpt-2/training_log.txt

@@ -0,0 +1,608 @@
+Step 0 | Loss: 11.070963 | Norm: 48.8176 | LR: 8.39e-07 | Time: 00:00:02 | Remaining: 00:01:39 | Avg Time/Step: 2.03
+Generated Text at Step 0: The king saidSeptemberSeptember 354 Fill ShameLots may>>>>>>>>umpyurry Apex nurses NEWS159 Vanguard FlemingictionTAJul Jihad LAR $\ underjri Columb
+Validating...
+Validation Loss: 10.916313171386719
+Step 1 | Loss: 11.171237 | Norm: 45.8637 | LR: 1.68e-06 | Time: 00:00:04 | Remaining: 00:01:41 | Avg Time/Step: 2.12
+Step 2 | Loss: 11.089214 | Norm: 49.5361 | LR: 2.52e-06 | Time: 00:00:04 | Remaining: 00:01:10 | Avg Time/Step: 1.50
+Validating...
+Validation Loss: 10.893363952636719
+Step 3 | Loss: 10.763819 | Norm: 52.8166 | LR: 3.36e-06 | Time: 00:00:05 | Remaining: 00:01:05 | Avg Time/Step: 1.41
+Step 4 | Loss: 11.204582 | Norm: 47.4927 | LR: 4.20e-06 | Time: 00:00:06 | Remaining: 00:00:54 | Avg Time/Step: 1.21
+Validating...
+Validation Loss: 10.86690902709961
+Step 5 | Loss: 10.957478 | Norm: 41.5032 | LR: 5.03e-06 | Time: 00:00:07 | Remaining: 00:00:58 | Avg Time/Step: 1.32
+Step 6 | Loss: 10.586459 | Norm: 43.5531 | LR: 5.87e-06 | Time: 00:00:08 | Remaining: 00:00:50 | Avg Time/Step: 1.18
+Validating...
+Validation Loss: 10.835768699645996
+Step 7 | Loss: 11.205253 | Norm: 44.9156 | LR: 6.71e-06 | Time: 00:00:09 | Remaining: 00:00:50 | Avg Time/Step: 1.20
+Step 8 | Loss: 10.609798 | Norm: 48.2627 | LR: 7.55e-06 | Time: 00:00:09 | Remaining: 00:00:44 | Avg Time/Step: 1.10
+Validating...
+Validation Loss: 10.792684555053711
+Step 9 | Loss: 9.896498 | Norm: 43.1797 | LR: 8.39e-06 | Time: 00:00:11 | Remaining: 00:00:44 | Avg Time/Step: 1.12
+Step 10 | Loss: 11.131380 | Norm: 44.4814 | LR: 9.23e-06 | Time: 00:00:11 | Remaining: 00:00:40 | Avg Time/Step: 1.04
+Validating...
+Validation Loss: 10.749573707580566
+Step 11 | Loss: 10.463729 | Norm: 47.8602 | LR: 1.01e-05 | Time: 00:00:12 | Remaining: 00:00:40 | Avg Time/Step: 1.06
+Step 12 | Loss: 10.880756 | Norm: 43.9313 | LR: 1.09e-05 | Time: 00:00:12 | Remaining: 00:00:36 | Avg Time/Step: 1.00
+Validating...
+Validation Loss: 10.712495803833008
+Step 13 | Loss: 9.864075 | Norm: 42.5331 | LR: 1.17e-05 | Time: 00:00:14 | Remaining: 00:00:36 | Avg Time/Step: 1.01
+Step 14 | Loss: 10.922160 | Norm: 44.6511 | LR: 1.26e-05 | Time: 00:00:14 | Remaining: 00:00:33 | Avg Time/Step: 0.96
+Validating...
+Validation Loss: 10.67584228515625
+Step 15 | Loss: 10.775851 | Norm: 44.4024 | LR: 1.34e-05 | Time: 00:00:15 | Remaining: 00:00:33 | Avg Time/Step: 0.98
+Step 16 | Loss: 10.330193 | Norm: 43.8886 | LR: 1.43e-05 | Time: 00:00:15 | Remaining: 00:00:30 | Avg Time/Step: 0.94
+Validating...
+Validation Loss: 10.615331649780273
+Step 17 | Loss: 10.270191 | Norm: 44.5217 | LR: 1.51e-05 | Time: 00:00:17 | Remaining: 00:00:30 | Avg Time/Step: 0.97
+Step 18 | Loss: 10.027596 | Norm: 46.1209 | LR: 1.59e-05 | Time: 00:00:17 | Remaining: 00:00:28 | Avg Time/Step: 0.93
+Validating...
+Validation Loss: 10.553497314453125
+Step 19 | Loss: 10.182181 | Norm: 40.7514 | LR: 1.68e-05 | Time: 00:00:19 | Remaining: 00:00:28 | Avg Time/Step: 0.96
+Step 20 | Loss: 9.555431 | Norm: 34.3714 | LR: 1.76e-05 | Time: 00:00:19 | Remaining: 00:00:26 | Avg Time/Step: 0.93
+Validating...
+Validation Loss: 10.458913803100586
+Step 21 | Loss: 10.136066 | Norm: 35.4013 | LR: 1.85e-05 | Time: 00:00:20 | Remaining: 00:00:26 | Avg Time/Step: 0.95
+Step 22 | Loss: 10.260824 | Norm: 35.9827 | LR: 1.93e-05 | Time: 00:00:21 | Remaining: 00:00:25 | Avg Time/Step: 0.93
+Validating...
+Validation Loss: 10.345619201660156
+Step 23 | Loss: 9.837000 | Norm: 34.4205 | LR: 2.01e-05 | Time: 00:00:22 | Remaining: 00:00:24 | Avg Time/Step: 0.94
+Step 24 | Loss: 10.418470 | Norm: 35.1306 | LR: 2.10e-05 | Time: 00:00:22 | Remaining: 00:00:22 | Avg Time/Step: 0.92
+Validating...
+Validation Loss: 10.242090225219727
+Step 25 | Loss: 10.759716 | Norm: 34.3984 | LR: 2.18e-05 | Time: 00:00:24 | Remaining: 00:00:22 | Avg Time/Step: 0.93
+Step 26 | Loss: 10.433059 | Norm: 33.6258 | LR: 2.27e-05 | Time: 00:00:24 | Remaining: 00:00:20 | Avg Time/Step: 0.91
+Validating...
+Validation Loss: 10.15864372253418
+Step 27 | Loss: 11.198073 | Norm: 33.6489 | LR: 2.35e-05 | Time: 00:00:25 | Remaining: 00:00:20 | Avg Time/Step: 0.93
+Step 28 | Loss: 9.453720 | Norm: 30.4983 | LR: 2.43e-05 | Time: 00:00:26 | Remaining: 00:00:18 | Avg Time/Step: 0.90
+Validating...
+Validation Loss: 10.089692115783691
+Step 29 | Loss: 10.043849 | Norm: 30.8429 | LR: 2.52e-05 | Time: 00:00:27 | Remaining: 00:00:18 | Avg Time/Step: 0.92
+Step 30 | Loss: 10.345837 | Norm: 28.3254 | LR: 2.60e-05 | Time: 00:00:27 | Remaining: 00:00:17 | Avg Time/Step: 0.90
+Validating...
+Validation Loss: 10.014737129211426
+Step 31 | Loss: 9.762772 | Norm: 28.7018 | LR: 2.69e-05 | Time: 00:00:29 | Remaining: 00:00:16 | Avg Time/Step: 0.92
+Step 32 | Loss: 9.099653 | Norm: 28.1757 | LR: 2.77e-05 | Time: 00:00:30 | Remaining: 00:00:15 | Avg Time/Step: 0.91
+Validating...
+Validation Loss: 9.956048011779785
+Step 33 | Loss: 8.908812 | Norm: 25.8786 | LR: 2.85e-05 | Time: 00:00:31 | Remaining: 00:00:14 | Avg Time/Step: 0.92
+Step 34 | Loss: 10.699462 | Norm: 25.3921 | LR: 2.94e-05 | Time: 00:00:31 | Remaining: 00:00:13 | Avg Time/Step: 0.90
+Validating...
+Validation Loss: 9.902624130249023
+Step 35 | Loss: 9.239347 | Norm: 25.3455 | LR: 3.02e-05 | Time: 00:00:33 | Remaining: 00:00:12 | Avg Time/Step: 0.92
+Step 36 | Loss: 10.142147 | Norm: 24.3786 | LR: 3.10e-05 | Time: 00:00:33 | Remaining: 00:00:11 | Avg Time/Step: 0.90
+Validating...
+Validation Loss: 9.841948509216309
+Step 37 | Loss: 10.260188 | Norm: 23.3623 | LR: 3.19e-05 | Time: 00:00:34 | Remaining: 00:00:10 | Avg Time/Step: 0.91
+Step 38 | Loss: 9.482347 | Norm: 24.0785 | LR: 3.27e-05 | Time: 00:00:35 | Remaining: 00:00:09 | Avg Time/Step: 0.90
+Validating...
+Validation Loss: 9.79233169555664
+Step 39 | Loss: 8.717162 | Norm: 23.1963 | LR: 3.36e-05 | Time: 00:00:36 | Remaining: 00:00:09 | Avg Time/Step: 0.91
+Step 40 | Loss: 9.536521 | Norm: 21.8829 | LR: 3.44e-05 | Time: 00:00:36 | Remaining: 00:00:08 | Avg Time/Step: 0.89
+Validating...
+Validation Loss: 9.746158599853516
+Step 41 | Loss: 9.760999 | Norm: 21.4380 | LR: 3.52e-05 | Time: 00:00:38 | Remaining: 00:00:07 | Avg Time/Step: 0.91
+Step 42 | Loss: 9.588884 | Norm: 22.2327 | LR: 3.61e-05 | Time: 00:00:38 | Remaining: 00:00:06 | Avg Time/Step: 0.89
+Validating...
+Validation Loss: 9.688400268554688
+Step 43 | Loss: 8.350541 | Norm: 20.6459 | LR: 3.69e-05 | Time: 00:00:39 | Remaining: 00:00:05 | Avg Time/Step: 0.90
+Step 44 | Loss: 9.594240 | Norm: 20.0493 | LR: 3.78e-05 | Time: 00:00:39 | Remaining: 00:00:04 | Avg Time/Step: 0.89
+Validating...
+Validation Loss: 9.622390747070312
+Step 45 | Loss: 8.240631 | Norm: 20.1186 | LR: 3.86e-05 | Time: 00:00:41 | Remaining: 00:00:03 | Avg Time/Step: 0.90
+Step 46 | Loss: 8.915052 | Norm: 20.4390 | LR: 3.94e-05 | Time: 00:00:41 | Remaining: 00:00:02 | Avg Time/Step: 0.88
+Validating...
+Validation Loss: 9.558349609375
+Step 47 | Loss: 8.285755 | Norm: 20.3787 | LR: 4.03e-05 | Time: 00:00:43 | Remaining: 00:00:01 | Avg Time/Step: 0.90
+Step 48 | Loss: 8.551549 | Norm: 20.1920 | LR: 4.11e-05 | Time: 00:00:43 | Remaining: 00:00:00 | Avg Time/Step: 0.89
+Validating...
+Validation Loss: 9.461584091186523
+Step 49 | Loss: 9.774352 | Norm: 20.2260 | LR: 4.20e-05 | Time: 00:00:45 | Remaining: 00:00:00 | Avg Time/Step: 0.91
+Step 0 | Loss: 11.070963 | Norm: 48.8176 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:46 | Avg Time/Step: 0.95
+Generated Text at Step 0: The king saidSeptemberSeptember 354 Fill ShameLots may>>>>>>>>umpyurry Apex nurses NEWS159 Vanguard FlemingictionTAJul Jihad LAR $\ underjri Columb
+Validating...
+Validation Loss: 10.916313171386719
+Step 1 | Loss: 11.171237 | Norm: 45.8637 | LR: 1.68e-06 | Time: 00:00:03 | Remaining: 00:01:12 | Avg Time/Step: 1.52
+Step 2 | Loss: 11.089214 | Norm: 49.5361 | LR: 2.52e-06 | Time: 00:00:03 | Remaining: 00:00:51 | Avg Time/Step: 1.09
+Validating...
+Validation Loss: 10.893363952636719
+Step 3 | Loss: 10.763819 | Norm: 52.8166 | LR: 3.36e-06 | Time: 00:00:04 | Remaining: 00:00:49 | Avg Time/Step: 1.08
+Step 4 | Loss: 11.204582 | Norm: 47.4927 | LR: 4.20e-06 | Time: 00:00:04 | Remaining: 00:00:41 | Avg Time/Step: 0.91
+Validating...
+Validation Loss: 10.86690902709961
+Step 5 | Loss: 10.957478 | Norm: 41.5032 | LR: 5.03e-06 | Time: 00:00:05 | Remaining: 00:00:40 | Avg Time/Step: 0.93
+Step 6 | Loss: 10.586459 | Norm: 43.5531 | LR: 5.87e-06 | Time: 00:00:05 | Remaining: 00:00:35 | Avg Time/Step: 0.83
+Validating...
+Validation Loss: 10.835768699645996
+Step 7 | Loss: 11.205253 | Norm: 44.9156 | LR: 6.71e-06 | Time: 00:00:07 | Remaining: 00:00:37 | Avg Time/Step: 0.89
+Step 8 | Loss: 10.609798 | Norm: 48.2627 | LR: 7.55e-06 | Time: 00:00:07 | Remaining: 00:00:33 | Avg Time/Step: 0.82
+Validating...
+Validation Loss: 10.792684555053711
+Step 9 | Loss: 9.896498 | Norm: 43.1797 | LR: 8.39e-06 | Time: 00:00:08 | Remaining: 00:00:35 | Avg Time/Step: 0.88
+Step 10 | Loss: 11.131380 | Norm: 44.4814 | LR: 9.23e-06 | Time: 00:00:09 | Remaining: 00:00:31 | Avg Time/Step: 0.82
+Validating...
+Validation Loss: 10.749573707580566
+Step 11 | Loss: 10.463729 | Norm: 47.8602 | LR: 1.01e-05 | Time: 00:00:10 | Remaining: 00:00:32 | Avg Time/Step: 0.86
+Step 12 | Loss: 10.880756 | Norm: 43.9313 | LR: 1.09e-05 | Time: 00:00:10 | Remaining: 00:00:30 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.712495803833008
+Step 13 | Loss: 9.864075 | Norm: 42.5331 | LR: 1.17e-05 | Time: 00:00:12 | Remaining: 00:00:30 | Avg Time/Step: 0.86
+Step 14 | Loss: 10.922160 | Norm: 44.6511 | LR: 1.26e-05 | Time: 00:00:12 | Remaining: 00:00:28 | Avg Time/Step: 0.82
+Validating...
+Validation Loss: 10.67584228515625
+Step 15 | Loss: 10.775851 | Norm: 44.4024 | LR: 1.34e-05 | Time: 00:00:13 | Remaining: 00:00:28 | Avg Time/Step: 0.85
+Step 16 | Loss: 10.330193 | Norm: 43.8886 | LR: 1.43e-05 | Time: 00:00:13 | Remaining: 00:00:26 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.615331649780273
+Step 17 | Loss: 10.270191 | Norm: 44.5217 | LR: 1.51e-05 | Time: 00:00:15 | Remaining: 00:00:26 | Avg Time/Step: 0.84
+Step 18 | Loss: 10.027596 | Norm: 46.1209 | LR: 1.59e-05 | Time: 00:00:15 | Remaining: 00:00:25 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.553497314453125
+Step 19 | Loss: 10.182181 | Norm: 40.7514 | LR: 1.68e-05 | Time: 00:00:16 | Remaining: 00:00:25 | Avg Time/Step: 0.84
+Step 20 | Loss: 9.555431 | Norm: 34.3714 | LR: 1.76e-05 | Time: 00:00:16 | Remaining: 00:00:23 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.458913803100586
+Step 21 | Loss: 10.136066 | Norm: 35.4013 | LR: 1.85e-05 | Time: 00:00:18 | Remaining: 00:00:23 | Avg Time/Step: 0.83
+Step 22 | Loss: 10.260824 | Norm: 35.9827 | LR: 1.93e-05 | Time: 00:00:18 | Remaining: 00:00:21 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 10.345619201660156
+Step 23 | Loss: 9.837000 | Norm: 34.4205 | LR: 2.01e-05 | Time: 00:00:19 | Remaining: 00:00:21 | Avg Time/Step: 0.82
+Step 24 | Loss: 10.418470 | Norm: 35.1306 | LR: 2.10e-05 | Time: 00:00:20 | Remaining: 00:00:20 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 10.242090225219727
+Step 25 | Loss: 10.759716 | Norm: 34.3984 | LR: 2.18e-05 | Time: 00:00:21 | Remaining: 00:00:19 | Avg Time/Step: 0.82
+Step 26 | Loss: 10.433059 | Norm: 33.6258 | LR: 2.27e-05 | Time: 00:00:21 | Remaining: 00:00:18 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 10.15864372253418
+Step 27 | Loss: 11.198073 | Norm: 33.6489 | LR: 2.35e-05 | Time: 00:00:22 | Remaining: 00:00:17 | Avg Time/Step: 0.81
+Step 28 | Loss: 9.453720 | Norm: 30.4983 | LR: 2.43e-05 | Time: 00:00:23 | Remaining: 00:00:16 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 10.089692115783691
+Step 29 | Loss: 10.043849 | Norm: 30.8429 | LR: 2.52e-05 | Time: 00:00:24 | Remaining: 00:00:16 | Avg Time/Step: 0.81
+Step 30 | Loss: 10.345837 | Norm: 28.3254 | LR: 2.60e-05 | Time: 00:00:24 | Remaining: 00:00:15 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 10.014737129211426
+Step 31 | Loss: 9.762772 | Norm: 28.7018 | LR: 2.69e-05 | Time: 00:00:25 | Remaining: 00:00:14 | Avg Time/Step: 0.81
+Step 32 | Loss: 9.099653 | Norm: 28.1757 | LR: 2.77e-05 | Time: 00:00:26 | Remaining: 00:00:13 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 9.956048011779785
+Step 33 | Loss: 8.908812 | Norm: 25.8786 | LR: 2.85e-05 | Time: 00:00:27 | Remaining: 00:00:12 | Avg Time/Step: 0.81
+Step 34 | Loss: 10.699462 | Norm: 25.3921 | LR: 2.94e-05 | Time: 00:00:27 | Remaining: 00:00:11 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 9.902624130249023
+Step 35 | Loss: 9.239347 | Norm: 25.3455 | LR: 3.02e-05 | Time: 00:00:28 | Remaining: 00:00:11 | Avg Time/Step: 0.80
+Step 36 | Loss: 10.142147 | Norm: 24.3786 | LR: 3.10e-05 | Time: 00:00:29 | Remaining: 00:00:10 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 9.841948509216309
+Step 37 | Loss: 10.260188 | Norm: 23.3623 | LR: 3.19e-05 | Time: 00:00:30 | Remaining: 00:00:09 | Avg Time/Step: 0.80
+Step 38 | Loss: 9.482347 | Norm: 24.0785 | LR: 3.27e-05 | Time: 00:00:30 | Remaining: 00:00:08 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 9.79233169555664
+Step 39 | Loss: 8.717162 | Norm: 23.1963 | LR: 3.36e-05 | Time: 00:00:32 | Remaining: 00:00:08 | Avg Time/Step: 0.81
+Step 40 | Loss: 9.536521 | Norm: 21.8829 | LR: 3.44e-05 | Time: 00:00:32 | Remaining: 00:00:07 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 9.746158599853516
+Step 41 | Loss: 9.760999 | Norm: 21.4380 | LR: 3.52e-05 | Time: 00:00:33 | Remaining: 00:00:06 | Avg Time/Step: 0.81
+Step 42 | Loss: 9.588884 | Norm: 22.2327 | LR: 3.61e-05 | Time: 00:00:34 | Remaining: 00:00:05 | Avg Time/Step: 0.79
+Validating...
+Validation Loss: 9.688400268554688
+Step 43 | Loss: 8.350541 | Norm: 20.6459 | LR: 3.69e-05 | Time: 00:00:35 | Remaining: 00:00:04 | Avg Time/Step: 0.81
+Step 44 | Loss: 9.594240 | Norm: 20.0493 | LR: 3.78e-05 | Time: 00:00:36 | Remaining: 00:00:04 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 9.622390747070312
+Step 45 | Loss: 8.240631 | Norm: 20.1186 | LR: 3.86e-05 | Time: 00:00:37 | Remaining: 00:00:03 | Avg Time/Step: 0.81
+Step 46 | Loss: 8.915052 | Norm: 20.4390 | LR: 3.94e-05 | Time: 00:00:37 | Remaining: 00:00:02 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 9.558349609375
+Step 47 | Loss: 8.285755 | Norm: 20.3787 | LR: 4.03e-05 | Time: 00:00:39 | Remaining: 00:00:01 | Avg Time/Step: 0.81
+Step 48 | Loss: 8.551549 | Norm: 20.1920 | LR: 4.11e-05 | Time: 00:00:39 | Remaining: 00:00:00 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 9.461584091186523
+Step 49 | Loss: 9.774352 | Norm: 20.2260 | LR: 4.20e-05 | Time: 00:00:40 | Remaining: 00:00:00 | Avg Time/Step: 0.81
+Step 0 | Loss: 11.070963 | Norm: 48.8176 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:44 | Avg Time/Step: 0.91
+Generated Text at Step 0: The king saidSeptemberSeptember 354 Fill ShameLots may>>>>>>>>umpyurry Apex nurses NEWS159 Vanguard FlemingictionTAJul Jihad LAR $\ underjri Columb
+Validating...
+Validation Loss: 10.916313171386719
+Step 1 | Loss: 11.171237 | Norm: 45.8637 | LR: 1.68e-06 | Time: 00:00:02 | Remaining: 00:01:11 | Avg Time/Step: 1.50
+Step 2 | Loss: 11.089214 | Norm: 49.5361 | LR: 2.52e-06 | Time: 00:00:03 | Remaining: 00:00:50 | Avg Time/Step: 1.07
+Validating...
+Validation Loss: 10.893363952636719
+Step 3 | Loss: 10.763819 | Norm: 52.8166 | LR: 3.36e-06 | Time: 00:00:04 | Remaining: 00:00:49 | Avg Time/Step: 1.08
+Step 4 | Loss: 11.204582 | Norm: 47.4927 | LR: 4.20e-06 | Time: 00:00:04 | Remaining: 00:00:40 | Avg Time/Step: 0.91
+Validating...
+Validation Loss: 10.86690902709961
+Step 5 | Loss: 10.957478 | Norm: 41.5032 | LR: 5.03e-06 | Time: 00:00:05 | Remaining: 00:00:40 | Avg Time/Step: 0.92
+Step 6 | Loss: 10.586459 | Norm: 43.5531 | LR: 5.87e-06 | Time: 00:00:05 | Remaining: 00:00:35 | Avg Time/Step: 0.83
+Validating...
+Validation Loss: 10.835768699645996
+Step 7 | Loss: 11.205253 | Norm: 44.9156 | LR: 6.71e-06 | Time: 00:00:07 | Remaining: 00:00:36 | Avg Time/Step: 0.88
+Step 8 | Loss: 10.609798 | Norm: 48.2627 | LR: 7.55e-06 | Time: 00:00:07 | Remaining: 00:00:33 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.792684555053711
+Step 9 | Loss: 9.896498 | Norm: 43.1797 | LR: 8.39e-06 | Time: 00:00:08 | Remaining: 00:00:34 | Avg Time/Step: 0.86
+Step 10 | Loss: 11.131380 | Norm: 44.4814 | LR: 9.23e-06 | Time: 00:00:08 | Remaining: 00:00:31 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 10.749573707580566
+Step 11 | Loss: 10.463729 | Norm: 47.8602 | LR: 1.01e-05 | Time: 00:00:10 | Remaining: 00:00:32 | Avg Time/Step: 0.84
+Step 12 | Loss: 10.880756 | Norm: 43.9313 | LR: 1.09e-05 | Time: 00:00:10 | Remaining: 00:00:29 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 10.712495803833008
+Step 13 | Loss: 9.864075 | Norm: 42.5331 | LR: 1.17e-05 | Time: 00:00:11 | Remaining: 00:00:30 | Avg Time/Step: 0.85
+Step 14 | Loss: 10.922160 | Norm: 44.6511 | LR: 1.26e-05 | Time: 00:00:12 | Remaining: 00:00:28 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.67584228515625
+Step 15 | Loss: 10.775851 | Norm: 44.4024 | LR: 1.34e-05 | Time: 00:00:13 | Remaining: 00:00:28 | Avg Time/Step: 0.84
+Step 16 | Loss: 10.330193 | Norm: 43.8886 | LR: 1.43e-05 | Time: 00:00:13 | Remaining: 00:00:26 | Avg Time/Step: 0.80
+Validating...
+Validation Loss: 10.615331649780273
+Step 17 | Loss: 10.270191 | Norm: 44.5217 | LR: 1.51e-05 | Time: 00:00:15 | Remaining: 00:00:27 | Avg Time/Step: 0.84
+Step 18 | Loss: 10.027596 | Norm: 46.1209 | LR: 1.59e-05 | Time: 00:00:15 | Remaining: 00:00:25 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.553497314453125
+Step 19 | Loss: 10.182181 | Norm: 40.7514 | LR: 1.68e-05 | Time: 00:00:16 | Remaining: 00:00:25 | Avg Time/Step: 0.85
+Step 20 | Loss: 9.555431 | Norm: 34.3714 | LR: 1.76e-05 | Time: 00:00:17 | Remaining: 00:00:23 | Avg Time/Step: 0.82
+Validating...
+Validation Loss: 10.458913803100586
+Step 21 | Loss: 10.136066 | Norm: 35.4013 | LR: 1.85e-05 | Time: 00:00:18 | Remaining: 00:00:23 | Avg Time/Step: 0.84
+Step 22 | Loss: 10.260824 | Norm: 35.9827 | LR: 1.93e-05 | Time: 00:00:18 | Remaining: 00:00:21 | Avg Time/Step: 0.81
+Validating...
+Validation Loss: 10.345619201660156
+Step 23 | Loss: 9.837000 | Norm: 34.4205 | LR: 2.01e-05 | Time: 00:00:20 | Remaining: 00:00:22 | Avg Time/Step: 0.85
+Step 24 | Loss: 10.418470 | Norm: 35.1306 | LR: 2.10e-05 | Time: 00:00:20 | Remaining: 00:00:20 | Avg Time/Step: 0.83
+Validating...
+Validation Loss: 10.242090225219727
+Step 25 | Loss: 10.759716 | Norm: 34.3984 | LR: 2.18e-05 | Time: 00:00:22 | Remaining: 00:00:20 | Avg Time/Step: 0.85
+Step 26 | Loss: 10.433059 | Norm: 33.6258 | LR: 2.27e-05 | Time: 00:00:22 | Remaining: 00:00:19 | Avg Time/Step: 0.85
+Validating...
+Validation Loss: 10.15864372253418
+Step 27 | Loss: 11.198073 | Norm: 33.6489 | LR: 2.35e-05 | Time: 00:00:24 | Remaining: 00:00:18 | Avg Time/Step: 0.86
+Step 28 | Loss: 9.453720 | Norm: 30.4983 | LR: 2.43e-05 | Time: 00:00:24 | Remaining: 00:00:17 | Avg Time/Step: 0.84
+Validating...
+Validation Loss: 10.089692115783691
+Step 29 | Loss: 10.043849 | Norm: 30.8429 | LR: 2.52e-05 | Time: 00:00:25 | Remaining: 00:00:17 | Avg Time/Step: 0.86
+Step 30 | Loss: 10.345837 | Norm: 28.3254 | LR: 2.60e-05 | Time: 00:00:26 | Remaining: 00:00:15 | Avg Time/Step: 0.84
+Validating...
+Validation Loss: 10.014737129211426
+Step 31 | Loss: 9.762772 | Norm: 28.7018 | LR: 2.69e-05 | Time: 00:00:27 | Remaining: 00:00:15 | Avg Time/Step: 0.86
+Step 32 | Loss: 9.099653 | Norm: 28.1757 | LR: 2.77e-05 | Time: 00:00:27 | Remaining: 00:00:14 | Avg Time/Step: 0.84
+Validating...
+Validation Loss: 9.956048011779785
+Step 33 | Loss: 8.908812 | Norm: 25.8786 | LR: 2.85e-05 | Time: 00:00:29 | Remaining: 00:00:13 | Avg Time/Step: 0.87
+Step 34 | Loss: 10.699462 | Norm: 25.3921 | LR: 2.94e-05 | Time: 00:00:29 | Remaining: 00:00:12 | Avg Time/Step: 0.85
+Validating...
+Validation Loss: 9.902624130249023
+Step 35 | Loss: 9.239347 | Norm: 25.3455 | LR: 3.02e-05 | Time: 00:00:31 | Remaining: 00:00:12 | Avg Time/Step: 0.87
+Step 36 | Loss: 10.142147 | Norm: 24.3786 | LR: 3.10e-05 | Time: 00:00:31 | Remaining: 00:00:11 | Avg Time/Step: 0.85
+Validating...
+Validation Loss: 9.841948509216309
+Step 37 | Loss: 10.260188 | Norm: 23.3623 | LR: 3.19e-05 | Time: 00:00:32 | Remaining: 00:00:10 | Avg Time/Step: 0.87
+Step 38 | Loss: 9.482347 | Norm: 24.0785 | LR: 3.27e-05 | Time: 00:00:33 | Remaining: 00:00:09 | Avg Time/Step: 0.85
+Validating...
+Validation Loss: 9.79233169555664
+Step 39 | Loss: 8.717162 | Norm: 23.1963 | LR: 3.36e-05 | Time: 00:00:34 | Remaining: 00:00:08 | Avg Time/Step: 0.86
+Step 40 | Loss: 9.536521 | Norm: 21.8829 | LR: 3.44e-05 | Time: 00:00:34 | Remaining: 00:00:07 | Avg Time/Step: 0.84
+Validating...
+Validation Loss: 9.746158599853516
+Step 41 | Loss: 9.760999 | Norm: 21.4380 | LR: 3.52e-05 | Time: 00:00:35 | Remaining: 00:00:06 | Avg Time/Step: 0.85
+Step 42 | Loss: 9.588884 | Norm: 22.2327 | LR: 3.61e-05 | Time: 00:00:36 | Remaining: 00:00:05 | Avg Time/Step: 0.84
+Validating...
+Validation Loss: 9.688400268554688
+Step 43 | Loss: 8.350541 | Norm: 20.6459 | LR: 3.69e-05 | Time: 00:00:37 | Remaining: 00:00:05 | Avg Time/Step: 0.85
+Step 44 | Loss: 9.594240 | Norm: 20.0493 | LR: 3.78e-05 | Time: 00:00:37 | Remaining: 00:00:04 | Avg Time/Step: 0.83
+Validating...
+Validation Loss: 9.622390747070312
+Step 45 | Loss: 8.240631 | Norm: 20.1186 | LR: 3.86e-05 | Time: 00:00:38 | Remaining: 00:00:03 | Avg Time/Step: 0.84
+Step 46 | Loss: 8.915052 | Norm: 20.4390 | LR: 3.94e-05 | Time: 00:00:39 | Remaining: 00:00:02 | Avg Time/Step: 0.83
+Validating...
+Validation Loss: 9.558349609375
+Step 47 | Loss: 8.285755 | Norm: 20.3787 | LR: 4.03e-05 | Time: 00:00:40 | Remaining: 00:00:01 | Avg Time/Step: 0.84
+Step 48 | Loss: 8.551549 | Norm: 20.1920 | LR: 4.11e-05 | Time: 00:00:40 | Remaining: 00:00:00 | Avg Time/Step: 0.83
+Validating...
+Validation Loss: 9.461584091186523
+Step 49 | Loss: 9.774352 | Norm: 20.2260 | LR: 4.20e-05 | Time: 00:00:42 | Remaining: 00:00:00 | Avg Time/Step: 0.84
+Step 0 | Loss: 11.070963 | Norm: 48.8176 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:42 | Avg Time/Step: 0.87
+Generated Text at Step 0: The king saidSeptemberSeptember 354 Fill ShameLots may>>>>>>>>umpyurry Apex nurses NEWS159 Vanguard FlemingictionTAJul Jihad LAR $\ underjri Columb
+Validating...
+Validation Loss: 10.916313171386719
+Step 1 | Loss: 11.171237 | Norm: 45.8637 | LR: 1.68e-06 | Time: 00:00:03 | Remaining: 00:01:12 | Avg Time/Step: 1.51
+Step 2 | Loss: 11.089214 | Norm: 49.5361 | LR: 2.52e-06 | Time: 00:00:03 | Remaining: 00:00:50 | Avg Time/Step: 1.08
+Step 3 | Loss: 10.763819 | Norm: 52.8166 | LR: 3.36e-06 | Time: 00:00:03 | Remaining: 00:00:40 | Avg Time/Step: 0.88
+Step 4 | Loss: 11.204582 | Norm: 47.4927 | LR: 4.20e-06 | Time: 00:00:04 | Remaining: 00:00:36 | Avg Time/Step: 0.81
+Step 5 | Loss: 10.957478 | Norm: 41.5032 | LR: 5.03e-06 | Time: 00:00:04 | Remaining: 00:00:32 | Avg Time/Step: 0.73
+Step 6 | Loss: 10.586459 | Norm: 43.5531 | LR: 5.87e-06 | Time: 00:00:04 | Remaining: 00:00:29 | Avg Time/Step: 0.68
+Step 7 | Loss: 11.205253 | Norm: 44.9156 | LR: 6.71e-06 | Time: 00:00:05 | Remaining: 00:00:26 | Avg Time/Step: 0.64
+Step 8 | Loss: 10.609798 | Norm: 48.2627 | LR: 7.55e-06 | Time: 00:00:05 | Remaining: 00:00:25 | Avg Time/Step: 0.61
+Step 9 | Loss: 9.896498 | Norm: 43.1797 | LR: 8.39e-06 | Time: 00:00:05 | Remaining: 00:00:23 | Avg Time/Step: 0.59
+Step 10 | Loss: 11.131380 | Norm: 44.4814 | LR: 9.23e-06 | Time: 00:00:06 | Remaining: 00:00:22 | Avg Time/Step: 0.57
+Validating...
+Validation Loss: 10.749573707580566
+Step 11 | Loss: 10.463729 | Norm: 47.8602 | LR: 1.01e-05 | Time: 00:00:07 | Remaining: 00:00:23 | Avg Time/Step: 0.62
+Step 12 | Loss: 10.880756 | Norm: 43.9313 | LR: 1.09e-05 | Time: 00:00:07 | Remaining: 00:00:22 | Avg Time/Step: 0.60
+Step 13 | Loss: 9.864075 | Norm: 42.5331 | LR: 1.17e-05 | Time: 00:00:08 | Remaining: 00:00:20 | Avg Time/Step: 0.57
+Step 14 | Loss: 10.922160 | Norm: 44.6511 | LR: 1.26e-05 | Time: 00:00:08 | Remaining: 00:00:19 | Avg Time/Step: 0.57
+Step 15 | Loss: 10.775851 | Norm: 44.4024 | LR: 1.34e-05 | Time: 00:00:08 | Remaining: 00:00:18 | Avg Time/Step: 0.55
+Step 16 | Loss: 10.330193 | Norm: 43.8886 | LR: 1.43e-05 | Time: 00:00:09 | Remaining: 00:00:18 | Avg Time/Step: 0.55
+Step 17 | Loss: 10.270191 | Norm: 44.5217 | LR: 1.51e-05 | Time: 00:00:09 | Remaining: 00:00:17 | Avg Time/Step: 0.53
+Step 18 | Loss: 10.027596 | Norm: 46.1209 | LR: 1.59e-05 | Time: 00:00:10 | Remaining: 00:00:16 | Avg Time/Step: 0.53
+Step 19 | Loss: 10.182181 | Norm: 40.7514 | LR: 1.68e-05 | Time: 00:00:10 | Remaining: 00:00:15 | Avg Time/Step: 0.52
+Step 20 | Loss: 9.555431 | Norm: 34.3714 | LR: 1.76e-05 | Time: 00:00:10 | Remaining: 00:00:14 | Avg Time/Step: 0.51
+Validating...
+Validation Loss: 10.458913803100586
+Step 21 | Loss: 10.136066 | Norm: 35.4013 | LR: 1.85e-05 | Time: 00:00:12 | Remaining: 00:00:15 | Avg Time/Step: 0.56
+Step 22 | Loss: 10.260824 | Norm: 35.9827 | LR: 1.93e-05 | Time: 00:00:12 | Remaining: 00:00:14 | Avg Time/Step: 0.54
+Step 23 | Loss: 9.837000 | Norm: 34.4205 | LR: 2.01e-05 | Time: 00:00:12 | Remaining: 00:00:14 | Avg Time/Step: 0.54
+Step 24 | Loss: 10.418470 | Norm: 35.1306 | LR: 2.10e-05 | Time: 00:00:13 | Remaining: 00:00:13 | Avg Time/Step: 0.53
+Step 25 | Loss: 10.759716 | Norm: 34.3984 | LR: 2.18e-05 | Time: 00:00:13 | Remaining: 00:00:12 | Avg Time/Step: 0.53
+Step 26 | Loss: 10.433059 | Norm: 33.6258 | LR: 2.27e-05 | Time: 00:00:14 | Remaining: 00:00:12 | Avg Time/Step: 0.53
+Step 27 | Loss: 11.198073 | Norm: 33.6489 | LR: 2.35e-05 | Time: 00:00:14 | Remaining: 00:00:11 | Avg Time/Step: 0.53
+Step 28 | Loss: 9.453720 | Norm: 30.4983 | LR: 2.43e-05 | Time: 00:00:15 | Remaining: 00:00:10 | Avg Time/Step: 0.52
+Step 29 | Loss: 10.043849 | Norm: 30.8429 | LR: 2.52e-05 | Time: 00:00:15 | Remaining: 00:00:10 | Avg Time/Step: 0.52
+Step 30 | Loss: 10.345837 | Norm: 28.3254 | LR: 2.60e-05 | Time: 00:00:16 | Remaining: 00:00:09 | Avg Time/Step: 0.52
+Validating...
+Validation Loss: 10.014737129211426
+Step 31 | Loss: 9.762772 | Norm: 28.7018 | LR: 2.69e-05 | Time: 00:00:17 | Remaining: 00:00:09 | Avg Time/Step: 0.54
+Step 32 | Loss: 9.099653 | Norm: 28.1757 | LR: 2.77e-05 | Time: 00:00:17 | Remaining: 00:00:09 | Avg Time/Step: 0.53
+Step 33 | Loss: 8.908812 | Norm: 25.8786 | LR: 2.85e-05 | Time: 00:00:17 | Remaining: 00:00:08 | Avg Time/Step: 0.53
+Step 34 | Loss: 10.699462 | Norm: 25.3921 | LR: 2.94e-05 | Time: 00:00:18 | Remaining: 00:00:07 | Avg Time/Step: 0.52
+Step 35 | Loss: 9.239347 | Norm: 25.3455 | LR: 3.02e-05 | Time: 00:00:18 | Remaining: 00:00:07 | Avg Time/Step: 0.52
+Step 36 | Loss: 10.142147 | Norm: 24.3786 | LR: 3.10e-05 | Time: 00:00:19 | Remaining: 00:00:06 | Avg Time/Step: 0.52
+Step 37 | Loss: 10.260188 | Norm: 23.3623 | LR: 3.19e-05 | Time: 00:00:19 | Remaining: 00:00:06 | Avg Time/Step: 0.52
+Step 38 | Loss: 9.482347 | Norm: 24.0785 | LR: 3.27e-05 | Time: 00:00:20 | Remaining: 00:00:05 | Avg Time/Step: 0.52
+Step 39 | Loss: 8.717162 | Norm: 23.1963 | LR: 3.36e-05 | Time: 00:00:20 | Remaining: 00:00:05 | Avg Time/Step: 0.52
+Step 40 | Loss: 9.536521 | Norm: 21.8829 | LR: 3.44e-05 | Time: 00:00:21 | Remaining: 00:00:04 | Avg Time/Step: 0.51
+Validating...
+Validation Loss: 9.746158599853516
+Step 41 | Loss: 9.760999 | Norm: 21.4380 | LR: 3.52e-05 | Time: 00:00:22 | Remaining: 00:00:04 | Avg Time/Step: 0.53
+Step 42 | Loss: 9.588884 | Norm: 22.2327 | LR: 3.61e-05 | Time: 00:00:22 | Remaining: 00:00:03 | Avg Time/Step: 0.53
+Step 43 | Loss: 8.350541 | Norm: 20.6459 | LR: 3.69e-05 | Time: 00:00:23 | Remaining: 00:00:03 | Avg Time/Step: 0.53
+Step 44 | Loss: 9.594240 | Norm: 20.0493 | LR: 3.78e-05 | Time: 00:00:23 | Remaining: 00:00:02 | Avg Time/Step: 0.52
+Step 45 | Loss: 8.240631 | Norm: 20.1186 | LR: 3.86e-05 | Time: 00:00:23 | Remaining: 00:00:02 | Avg Time/Step: 0.52
+Step 46 | Loss: 8.915052 | Norm: 20.4390 | LR: 3.94e-05 | Time: 00:00:24 | Remaining: 00:00:01 | Avg Time/Step: 0.51
+Step 47 | Loss: 8.285755 | Norm: 20.3787 | LR: 4.03e-05 | Time: 00:00:24 | Remaining: 00:00:01 | Avg Time/Step: 0.51
+Step 48 | Loss: 8.551549 | Norm: 20.1920 | LR: 4.11e-05 | Time: 00:00:24 | Remaining: 00:00:00 | Avg Time/Step: 0.51
+Step 49 | Loss: 9.774352 | Norm: 20.2260 | LR: 4.20e-05 | Time: 00:00:25 | Remaining: 00:00:00 | Avg Time/Step: 0.51
+Step 0 | Loss: 11.633898 | Norm: 44.3633 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:42 | Avg Time/Step: 0.86
+Generated Text at Step 0: The king said ginger stupid 194 idi shrugged outperPLIC Pitch chapter chapter 169 Drac darkest darkesttic Suk encrypted outperGraphics bisexual PitchBC1987 Cobra drives
+Validating...
+Validation Loss: 10.924361228942871
+Step 1 | Loss: 10.875149 | Norm: 51.7900 | LR: 1.68e-06 | Time: 00:00:02 | Remaining: 00:01:09 | Avg Time/Step: 1.45
+Step 2 | Loss: 10.981276 | Norm: 44.8046 | LR: 2.52e-06 | Time: 00:00:03 | Remaining: 00:00:49 | Avg Time/Step: 1.06
+Step 3 | Loss: 10.517224 | Norm: 49.3383 | LR: 3.36e-06 | Time: 00:00:03 | Remaining: 00:00:39 | Avg Time/Step: 0.86
+Step 4 | Loss: 11.220371 | Norm: 50.4130 | LR: 4.20e-06 | Time: 00:00:03 | Remaining: 00:00:32 | Avg Time/Step: 0.73
+Step 5 | Loss: 11.176923 | Norm: 47.4072 | LR: 5.03e-06 | Time: 00:00:03 | Remaining: 00:00:28 | Avg Time/Step: 0.65
+Step 6 | Loss: 10.935453 | Norm: 45.3805 | LR: 5.87e-06 | Time: 00:00:04 | Remaining: 00:00:25 | Avg Time/Step: 0.59
+Step 7 | Loss: 10.582232 | Norm: 46.3087 | LR: 6.71e-06 | Time: 00:00:04 | Remaining: 00:00:23 | Avg Time/Step: 0.56
+Step 8 | Loss: 11.022345 | Norm: 43.2213 | LR: 7.55e-06 | Time: 00:00:04 | Remaining: 00:00:22 | Avg Time/Step: 0.55
+Step 9 | Loss: 10.926727 | Norm: 44.3769 | LR: 8.39e-06 | Time: 00:00:05 | Remaining: 00:00:21 | Avg Time/Step: 0.53
+Step 10 | Loss: 10.986204 | Norm: 45.1195 | LR: 9.23e-06 | Time: 00:00:05 | Remaining: 00:00:20 | Avg Time/Step: 0.52
+Validating...
+Validation Loss: 10.735955238342285
+Step 11 | Loss: 11.179207 | Norm: 47.9544 | LR: 1.01e-05 | Time: 00:00:07 | Remaining: 00:00:22 | Avg Time/Step: 0.59
+Step 12 | Loss: 10.763081 | Norm: 43.9656 | LR: 1.09e-05 | Time: 00:00:07 | Remaining: 00:00:20 | Avg Time/Step: 0.56
+Step 13 | Loss: 10.720469 | Norm: 43.3385 | LR: 1.17e-05 | Time: 00:00:07 | Remaining: 00:00:19 | Avg Time/Step: 0.54
+Step 14 | Loss: 11.064083 | Norm: 43.1475 | LR: 1.26e-05 | Time: 00:00:07 | Remaining: 00:00:18 | Avg Time/Step: 0.53
+Step 15 | Loss: 10.534277 | Norm: 44.0213 | LR: 1.34e-05 | Time: 00:00:08 | Remaining: 00:00:17 | Avg Time/Step: 0.52
+Step 16 | Loss: 10.638024 | Norm: 42.8165 | LR: 1.43e-05 | Time: 00:00:08 | Remaining: 00:00:16 | Avg Time/Step: 0.51
+Step 17 | Loss: 11.247206 | Norm: 39.1321 | LR: 1.51e-05 | Time: 00:00:09 | Remaining: 00:00:16 | Avg Time/Step: 0.51
+Step 18 | Loss: 10.615942 | Norm: 45.8611 | LR: 1.59e-05 | Time: 00:00:09 | Remaining: 00:00:15 | Avg Time/Step: 0.50
+Step 19 | Loss: 10.434818 | Norm: 35.2029 | LR: 1.68e-05 | Time: 00:00:09 | Remaining: 00:00:14 | Avg Time/Step: 0.49
+Step 20 | Loss: 9.872961 | Norm: 37.0101 | LR: 1.76e-05 | Time: 00:00:10 | Remaining: 00:00:14 | Avg Time/Step: 0.49
+Validating...
+Validation Loss: 10.45177936553955
+Step 21 | Loss: 10.303642 | Norm: 38.1966 | LR: 1.85e-05 | Time: 00:00:11 | Remaining: 00:00:14 | Avg Time/Step: 0.53
+Step 22 | Loss: 10.344124 | Norm: 36.7267 | LR: 1.93e-05 | Time: 00:00:11 | Remaining: 00:00:13 | Avg Time/Step: 0.51
+Step 23 | Loss: 10.358528 | Norm: 33.4473 | LR: 2.01e-05 | Time: 00:00:12 | Remaining: 00:00:13 | Avg Time/Step: 0.50
+Step 24 | Loss: 10.899721 | Norm: 33.1147 | LR: 2.10e-05 | Time: 00:00:12 | Remaining: 00:00:12 | Avg Time/Step: 0.50
+Step 25 | Loss: 10.167845 | Norm: 32.0061 | LR: 2.18e-05 | Time: 00:00:12 | Remaining: 00:00:11 | Avg Time/Step: 0.50
+Step 26 | Loss: 10.658374 | Norm: 32.8027 | LR: 2.27e-05 | Time: 00:00:13 | Remaining: 00:00:11 | Avg Time/Step: 0.49
+Step 27 | Loss: 11.409204 | Norm: 32.2853 | LR: 2.35e-05 | Time: 00:00:13 | Remaining: 00:00:10 | Avg Time/Step: 0.49
+Step 28 | Loss: 9.699551 | Norm: 28.3168 | LR: 2.43e-05 | Time: 00:00:14 | Remaining: 00:00:10 | Avg Time/Step: 0.48
+Step 29 | Loss: 10.293508 | Norm: 29.6286 | LR: 2.52e-05 | Time: 00:00:14 | Remaining: 00:00:09 | Avg Time/Step: 0.48
+Step 30 | Loss: 10.796824 | Norm: 32.4335 | LR: 2.60e-05 | Time: 00:00:14 | Remaining: 00:00:09 | Avg Time/Step: 0.48
+Validating...
+Validation Loss: 10.094558715820312
+Step 31 | Loss: 9.871226 | Norm: 29.4167 | LR: 2.69e-05 | Time: 00:00:16 | Remaining: 00:00:09 | Avg Time/Step: 0.50
+Step 32 | Loss: 9.355142 | Norm: 29.9377 | LR: 2.77e-05 | Time: 00:00:16 | Remaining: 00:00:08 | Avg Time/Step: 0.49
+Step 33 | Loss: 9.169601 | Norm: 28.7526 | LR: 2.85e-05 | Time: 00:00:16 | Remaining: 00:00:07 | Avg Time/Step: 0.49
+Step 34 | Loss: 11.027575 | Norm: 25.0330 | LR: 2.94e-05 | Time: 00:00:17 | Remaining: 00:00:07 | Avg Time/Step: 0.49
+Step 35 | Loss: 9.624268 | Norm: 26.2367 | LR: 3.02e-05 | Time: 00:00:17 | Remaining: 00:00:06 | Avg Time/Step: 0.48
+Step 36 | Loss: 10.801857 | Norm: 26.3915 | LR: 3.10e-05 | Time: 00:00:17 | Remaining: 00:00:06 | Avg Time/Step: 0.48
+Step 37 | Loss: 10.625546 | Norm: 24.5588 | LR: 3.19e-05 | Time: 00:00:18 | Remaining: 00:00:05 | Avg Time/Step: 0.48
+Step 38 | Loss: 9.325054 | Norm: 23.9140 | LR: 3.27e-05 | Time: 00:00:18 | Remaining: 00:00:05 | Avg Time/Step: 0.48
+Step 39 | Loss: 8.672618 | Norm: 22.6858 | LR: 3.36e-05 | Time: 00:00:18 | Remaining: 00:00:04 | Avg Time/Step: 0.47
+Step 40 | Loss: 9.316482 | Norm: 23.7950 | LR: 3.44e-05 | Time: 00:00:19 | Remaining: 00:00:04 | Avg Time/Step: 0.47
+Validating...
+Validation Loss: 9.847529411315918
+Step 41 | Loss: 9.895099 | Norm: 22.9021 | LR: 3.52e-05 | Time: 00:00:20 | Remaining: 00:00:03 | Avg Time/Step: 0.49
+Step 42 | Loss: 9.908270 | Norm: 22.2757 | LR: 3.61e-05 | Time: 00:00:20 | Remaining: 00:00:03 | Avg Time/Step: 0.48
+Step 43 | Loss: 8.863647 | Norm: 21.6877 | LR: 3.69e-05 | Time: 00:00:21 | Remaining: 00:00:02 | Avg Time/Step: 0.48
+Step 44 | Loss: 9.615014 | Norm: 22.1502 | LR: 3.78e-05 | Time: 00:00:21 | Remaining: 00:00:02 | Avg Time/Step: 0.48
+Step 45 | Loss: 7.558504 | Norm: 20.6337 | LR: 3.86e-05 | Time: 00:00:21 | Remaining: 00:00:01 | Avg Time/Step: 0.48
+Step 46 | Loss: 9.626184 | Norm: 22.2072 | LR: 3.94e-05 | Time: 00:00:22 | Remaining: 00:00:01 | Avg Time/Step: 0.47
+Step 47 | Loss: 8.649675 | Norm: 21.2089 | LR: 4.03e-05 | Time: 00:00:22 | Remaining: 00:00:00 | Avg Time/Step: 0.47
+Step 48 | Loss: 8.570056 | Norm: 21.1816 | LR: 4.11e-05 | Time: 00:00:23 | Remaining: 00:00:00 | Avg Time/Step: 0.47
+Step 49 | Loss: 9.796856 | Norm: 20.8208 | LR: 4.20e-05 | Time: 00:00:23 | Remaining: 00:00:00 | Avg Time/Step: 0.47
+Step 0 | Loss: 11.633898 | Norm: 44.3633 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:44 | Avg Time/Step: 0.91
+Generated Text at Step 0: The king said ginger stupid 194 idi shrugged outperPLIC Pitch chapter chapter 169 Drac darkest darkesttic Suk encrypted outperGraphics bisexual PitchBC1987 Cobra drives
+Validating...
+Validation Loss: 10.924361228942871
+Step 1 | Loss: 10.875149 | Norm: 51.7900 | LR: 1.68e-06 | Time: 00:00:03 | Remaining: 00:01:34 | Avg Time/Step: 1.97
+Step 2 | Loss: 10.981276 | Norm: 44.8046 | LR: 2.52e-06 | Time: 00:00:04 | Remaining: 00:01:05 | Avg Time/Step: 1.39
+Step 3 | Loss: 10.517224 | Norm: 49.3383 | LR: 3.36e-06 | Time: 00:00:04 | Remaining: 00:00:50 | Avg Time/Step: 1.10
+Step 4 | Loss: 11.220371 | Norm: 50.4130 | LR: 4.20e-06 | Time: 00:00:04 | Remaining: 00:00:41 | Avg Time/Step: 0.92
+Step 5 | Loss: 11.176923 | Norm: 47.4072 | LR: 5.03e-06 | Time: 00:00:04 | Remaining: 00:00:35 | Avg Time/Step: 0.80
+Step 6 | Loss: 10.935453 | Norm: 45.3805 | LR: 5.87e-06 | Time: 00:00:05 | Remaining: 00:00:31 | Avg Time/Step: 0.72
+Step 7 | Loss: 10.582232 | Norm: 46.3087 | LR: 6.71e-06 | Time: 00:00:05 | Remaining: 00:00:27 | Avg Time/Step: 0.66
+Step 8 | Loss: 11.022345 | Norm: 43.2213 | LR: 7.55e-06 | Time: 00:00:05 | Remaining: 00:00:25 | Avg Time/Step: 0.62
+Step 9 | Loss: 10.926727 | Norm: 44.3769 | LR: 8.39e-06 | Time: 00:00:06 | Remaining: 00:00:24 | Avg Time/Step: 0.61
+Step 10 | Loss: 10.986204 | Norm: 45.1195 | LR: 9.23e-06 | Time: 00:00:06 | Remaining: 00:00:22 | Avg Time/Step: 0.59
+Validating...
+Validation Loss: 10.735955238342285
+Step 11 | Loss: 11.179207 | Norm: 47.9544 | LR: 1.01e-05 | Time: 00:00:07 | Remaining: 00:00:25 | Avg Time/Step: 0.66
+Step 12 | Loss: 10.763081 | Norm: 43.9656 | LR: 1.09e-05 | Time: 00:00:08 | Remaining: 00:00:23 | Avg Time/Step: 0.63
+Step 13 | Loss: 10.720469 | Norm: 43.3385 | LR: 1.17e-05 | Time: 00:00:08 | Remaining: 00:00:22 | Avg Time/Step: 0.63
+Step 14 | Loss: 11.064083 | Norm: 43.1475 | LR: 1.26e-05 | Time: 00:00:09 | Remaining: 00:00:21 | Avg Time/Step: 0.62
+Step 15 | Loss: 10.534277 | Norm: 44.0213 | LR: 1.34e-05 | Time: 00:00:09 | Remaining: 00:00:20 | Avg Time/Step: 0.60
+Step 16 | Loss: 10.638024 | Norm: 42.8165 | LR: 1.43e-05 | Time: 00:00:10 | Remaining: 00:00:20 | Avg Time/Step: 0.61
+Step 17 | Loss: 11.247206 | Norm: 39.1321 | LR: 1.51e-05 | Time: 00:00:10 | Remaining: 00:00:18 | Avg Time/Step: 0.59
+Step 18 | Loss: 10.615942 | Norm: 45.8611 | LR: 1.59e-05 | Time: 00:00:11 | Remaining: 00:00:18 | Avg Time/Step: 0.59
+Step 19 | Loss: 10.434818 | Norm: 35.2029 | LR: 1.68e-05 | Time: 00:00:11 | Remaining: 00:00:17 | Avg Time/Step: 0.58
+Step 20 | Loss: 9.872961 | Norm: 37.0101 | LR: 1.76e-05 | Time: 00:00:11 | Remaining: 00:00:16 | Avg Time/Step: 0.57
+Validating...
+Validation Loss: 10.45177936553955
+Step 21 | Loss: 10.303642 | Norm: 38.1966 | LR: 1.85e-05 | Time: 00:00:13 | Remaining: 00:00:16 | Avg Time/Step: 0.60
+Step 22 | Loss: 10.344124 | Norm: 36.7267 | LR: 1.93e-05 | Time: 00:00:13 | Remaining: 00:00:15 | Avg Time/Step: 0.59
+Step 23 | Loss: 10.358528 | Norm: 33.4473 | LR: 2.01e-05 | Time: 00:00:14 | Remaining: 00:00:15 | Avg Time/Step: 0.59
+Step 24 | Loss: 10.899721 | Norm: 33.1147 | LR: 2.10e-05 | Time: 00:00:14 | Remaining: 00:00:14 | Avg Time/Step: 0.59
+Step 25 | Loss: 10.167845 | Norm: 32.0061 | LR: 2.18e-05 | Time: 00:00:14 | Remaining: 00:00:13 | Avg Time/Step: 0.58
+Step 26 | Loss: 10.658374 | Norm: 32.8027 | LR: 2.27e-05 | Time: 00:00:15 | Remaining: 00:00:13 | Avg Time/Step: 0.59
+Step 27 | Loss: 11.409204 | Norm: 32.2853 | LR: 2.35e-05 | Time: 00:00:16 | Remaining: 00:00:12 | Avg Time/Step: 0.58
+Step 28 | Loss: 9.699551 | Norm: 28.3168 | LR: 2.43e-05 | Time: 00:00:16 | Remaining: 00:00:12 | Avg Time/Step: 0.58
+Step 29 | Loss: 10.293508 | Norm: 29.6286 | LR: 2.52e-05 | Time: 00:00:17 | Remaining: 00:00:11 | Avg Time/Step: 0.58
+Step 30 | Loss: 10.796824 | Norm: 32.4335 | LR: 2.60e-05 | Time: 00:00:17 | Remaining: 00:00:10 | Avg Time/Step: 0.57
+Validating...
+Validation Loss: 10.094558715820312
+Step 31 | Loss: 9.871226 | Norm: 29.4167 | LR: 2.69e-05 | Time: 00:00:19 | Remaining: 00:00:10 | Avg Time/Step: 0.60
+Step 32 | Loss: 9.355142 | Norm: 29.9377 | LR: 2.77e-05 | Time: 00:00:19 | Remaining: 00:00:09 | Avg Time/Step: 0.58
+Step 33 | Loss: 9.169601 | Norm: 28.7526 | LR: 2.85e-05 | Time: 00:00:19 | Remaining: 00:00:09 | Avg Time/Step: 0.58
+Step 34 | Loss: 11.027575 | Norm: 25.0330 | LR: 2.94e-05 | Time: 00:00:20 | Remaining: 00:00:08 | Avg Time/Step: 0.58
+Step 35 | Loss: 9.624268 | Norm: 26.2367 | LR: 3.02e-05 | Time: 00:00:20 | Remaining: 00:00:08 | Avg Time/Step: 0.58
+Step 36 | Loss: 10.801857 | Norm: 26.3915 | LR: 3.10e-05 | Time: 00:00:21 | Remaining: 00:00:07 | Avg Time/Step: 0.58
+Step 37 | Loss: 10.625546 | Norm: 24.5588 | LR: 3.19e-05 | Time: 00:00:22 | Remaining: 00:00:06 | Avg Time/Step: 0.58
+Step 38 | Loss: 9.325054 | Norm: 23.9140 | LR: 3.27e-05 | Time: 00:00:22 | Remaining: 00:00:06 | Avg Time/Step: 0.57
+Step 39 | Loss: 8.672618 | Norm: 22.6858 | LR: 3.36e-05 | Time: 00:00:22 | Remaining: 00:00:05 | Avg Time/Step: 0.57
+Step 40 | Loss: 9.316482 | Norm: 23.7950 | LR: 3.44e-05 | Time: 00:00:23 | Remaining: 00:00:05 | Avg Time/Step: 0.56
+Validating...
+Validation Loss: 9.847529411315918
+Step 41 | Loss: 9.895099 | Norm: 22.9021 | LR: 3.52e-05 | Time: 00:00:24 | Remaining: 00:00:04 | Avg Time/Step: 0.58
+Step 42 | Loss: 9.908270 | Norm: 22.2757 | LR: 3.61e-05 | Time: 00:00:24 | Remaining: 00:00:04 | Avg Time/Step: 0.57
+Step 43 | Loss: 8.863647 | Norm: 21.6877 | LR: 3.69e-05 | Time: 00:00:25 | Remaining: 00:00:03 | Avg Time/Step: 0.57
+Step 44 | Loss: 9.615014 | Norm: 22.1502 | LR: 3.78e-05 | Time: 00:00:25 | Remaining: 00:00:02 | Avg Time/Step: 0.57
+Step 45 | Loss: 7.558504 | Norm: 20.6337 | LR: 3.86e-05 | Time: 00:00:25 | Remaining: 00:00:02 | Avg Time/Step: 0.56
+Step 46 | Loss: 9.626184 | Norm: 22.2072 | LR: 3.94e-05 | Time: 00:00:26 | Remaining: 00:00:01 | Avg Time/Step: 0.56
+Step 47 | Loss: 8.649675 | Norm: 21.2089 | LR: 4.03e-05 | Time: 00:00:26 | Remaining: 00:00:01 | Avg Time/Step: 0.55
+Step 48 | Loss: 8.570056 | Norm: 21.1816 | LR: 4.11e-05 | Time: 00:00:27 | Remaining: 00:00:00 | Avg Time/Step: 0.55
+Step 49 | Loss: 9.796856 | Norm: 20.8208 | LR: 4.20e-05 | Time: 00:00:27 | Remaining: 00:00:00 | Avg Time/Step: 0.55
+Step 0 | Loss: 11.633898 | Norm: 44.3633 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:42 | Avg Time/Step: 0.86
+Generated Text at Step 0: The king said ginger stupid 194 idi shrugged outperPLIC Pitch chapter chapter 169 Drac darkest darkesttic Suk encrypted outperGraphics bisexual PitchBC1987 Cobra drives
+Validating...
+Validation Loss: 10.924361228942871
+Step 1 | Loss: 10.875149 | Norm: 51.7900 | LR: 1.68e-06 | Time: 00:00:03 | Remaining: 00:01:29 | Avg Time/Step: 1.87
+Step 2 | Loss: 10.981276 | Norm: 44.8046 | LR: 2.52e-06 | Time: 00:00:03 | Remaining: 00:01:01 | Avg Time/Step: 1.32
+Step 3 | Loss: 10.517224 | Norm: 49.3383 | LR: 3.36e-06 | Time: 00:00:04 | Remaining: 00:00:48 | Avg Time/Step: 1.04
+Step 4 | Loss: 11.220371 | Norm: 50.4130 | LR: 4.20e-06 | Time: 00:00:04 | Remaining: 00:00:39 | Avg Time/Step: 0.88
+Step 5 | Loss: 11.176923 | Norm: 47.4072 | LR: 5.03e-06 | Time: 00:00:04 | Remaining: 00:00:33 | Avg Time/Step: 0.77
+Step 6 | Loss: 10.935453 | Norm: 45.3805 | LR: 5.87e-06 | Time: 00:00:05 | Remaining: 00:00:35 | Avg Time/Step: 0.82
+Step 7 | Loss: 10.582232 | Norm: 46.3087 | LR: 6.71e-06 | Time: 00:00:05 | Remaining: 00:00:31 | Avg Time/Step: 0.74
+Step 8 | Loss: 11.022345 | Norm: 43.2213 | LR: 7.55e-06 | Time: 00:00:06 | Remaining: 00:00:28 | Avg Time/Step: 0.68
+Step 9 | Loss: 10.926727 | Norm: 44.3769 | LR: 8.39e-06 | Time: 00:00:06 | Remaining: 00:00:25 | Avg Time/Step: 0.64
+Step 10 | Loss: 10.986204 | Norm: 45.1195 | LR: 9.23e-06 | Time: 00:00:06 | Remaining: 00:00:23 | Avg Time/Step: 0.60
+Validating...
+Validation Loss: 10.735955238342285
+Step 11 | Loss: 11.179207 | Norm: 47.9544 | LR: 1.01e-05 | Time: 00:00:08 | Remaining: 00:00:27 | Avg Time/Step: 0.72
+Step 12 | Loss: 10.763081 | Norm: 43.9656 | LR: 1.09e-05 | Time: 00:00:08 | Remaining: 00:00:25 | Avg Time/Step: 0.68
+Step 13 | Loss: 10.720469 | Norm: 43.3385 | LR: 1.17e-05 | Time: 00:00:09 | Remaining: 00:00:23 | Avg Time/Step: 0.65
+Step 14 | Loss: 11.064083 | Norm: 43.1475 | LR: 1.26e-05 | Time: 00:00:09 | Remaining: 00:00:22 | Avg Time/Step: 0.64
+Step 15 | Loss: 10.534277 | Norm: 44.0213 | LR: 1.34e-05 | Time: 00:00:09 | Remaining: 00:00:21 | Avg Time/Step: 0.62
+Step 16 | Loss: 10.638024 | Norm: 42.8165 | LR: 1.43e-05 | Time: 00:00:11 | Remaining: 00:00:22 | Avg Time/Step: 0.67
+Step 17 | Loss: 11.247206 | Norm: 39.1321 | LR: 1.51e-05 | Time: 00:00:11 | Remaining: 00:00:20 | Avg Time/Step: 0.65
+Step 18 | Loss: 10.615942 | Norm: 45.8611 | LR: 1.59e-05 | Time: 00:00:12 | Remaining: 00:00:19 | Avg Time/Step: 0.64
+Step 19 | Loss: 10.434818 | Norm: 35.2029 | LR: 1.68e-05 | Time: 00:00:12 | Remaining: 00:00:19 | Avg Time/Step: 0.64
+Step 20 | Loss: 9.872961 | Norm: 37.0101 | LR: 1.76e-05 | Time: 00:00:13 | Remaining: 00:00:18 | Avg Time/Step: 0.63
+Validating...
+Validation Loss: 10.45177936553955
+Step 21 | Loss: 10.303642 | Norm: 38.1966 | LR: 1.85e-05 | Time: 00:00:15 | Remaining: 00:00:19 | Avg Time/Step: 0.70
+Step 22 | Loss: 10.344124 | Norm: 36.7267 | LR: 1.93e-05 | Time: 00:00:15 | Remaining: 00:00:18 | Avg Time/Step: 0.68
+Step 23 | Loss: 10.358528 | Norm: 33.4473 | LR: 2.01e-05 | Time: 00:00:16 | Remaining: 00:00:17 | Avg Time/Step: 0.67
+Step 24 | Loss: 10.899721 | Norm: 33.1147 | LR: 2.10e-05 | Time: 00:00:16 | Remaining: 00:00:16 | Avg Time/Step: 0.66
+Step 25 | Loss: 10.167845 | Norm: 32.0061 | LR: 2.18e-05 | Time: 00:00:16 | Remaining: 00:00:15 | Avg Time/Step: 0.65
+Step 26 | Loss: 10.658374 | Norm: 32.8027 | LR: 2.27e-05 | Time: 00:00:18 | Remaining: 00:00:15 | Avg Time/Step: 0.68
+Step 27 | Loss: 11.409204 | Norm: 32.2853 | LR: 2.35e-05 | Time: 00:00:18 | Remaining: 00:00:14 | Avg Time/Step: 0.66
+Step 28 | Loss: 9.699551 | Norm: 28.3168 | LR: 2.43e-05 | Time: 00:00:18 | Remaining: 00:00:13 | Avg Time/Step: 0.65
+Step 29 | Loss: 10.293508 | Norm: 29.6286 | LR: 2.52e-05 | Time: 00:00:19 | Remaining: 00:00:12 | Avg Time/Step: 0.64
+Step 30 | Loss: 10.796824 | Norm: 32.4335 | LR: 2.60e-05 | Time: 00:00:19 | Remaining: 00:00:12 | Avg Time/Step: 0.63
+Validating...
+Validation Loss: 10.094558715820312
+Step 31 | Loss: 9.871226 | Norm: 29.4167 | LR: 2.69e-05 | Time: 00:00:21 | Remaining: 00:00:12 | Avg Time/Step: 0.68
+Step 32 | Loss: 9.355142 | Norm: 29.9377 | LR: 2.77e-05 | Time: 00:00:22 | Remaining: 00:00:11 | Avg Time/Step: 0.67
+Step 33 | Loss: 9.169601 | Norm: 28.7526 | LR: 2.85e-05 | Time: 00:00:22 | Remaining: 00:00:10 | Avg Time/Step: 0.66
+Step 34 | Loss: 11.027575 | Norm: 25.0330 | LR: 2.94e-05 | Time: 00:00:22 | Remaining: 00:00:09 | Avg Time/Step: 0.65
+Step 35 | Loss: 9.624268 | Norm: 26.2367 | LR: 3.02e-05 | Time: 00:00:23 | Remaining: 00:00:08 | Avg Time/Step: 0.64
+Step 36 | Loss: 10.801857 | Norm: 26.3915 | LR: 3.10e-05 | Time: 00:00:24 | Remaining: 00:00:08 | Avg Time/Step: 0.66
+Step 37 | Loss: 10.625546 | Norm: 24.5588 | LR: 3.19e-05 | Time: 00:00:24 | Remaining: 00:00:07 | Avg Time/Step: 0.66
+Step 38 | Loss: 9.325054 | Norm: 23.9140 | LR: 3.27e-05 | Time: 00:00:25 | Remaining: 00:00:07 | Avg Time/Step: 0.65
+Step 39 | Loss: 8.672618 | Norm: 22.6858 | LR: 3.36e-05 | Time: 00:00:26 | Remaining: 00:00:06 | Avg Time/Step: 0.65
+Step 40 | Loss: 9.316482 | Norm: 23.7950 | LR: 3.44e-05 | Time: 00:00:26 | Remaining: 00:00:05 | Avg Time/Step: 0.65
+Validating...
+Validation Loss: 9.847529411315918
+Step 41 | Loss: 9.895099 | Norm: 22.9021 | LR: 3.52e-05 | Time: 00:00:28 | Remaining: 00:00:05 | Avg Time/Step: 0.68
+Step 42 | Loss: 9.908270 | Norm: 22.2757 | LR: 3.61e-05 | Time: 00:00:28 | Remaining: 00:00:04 | Avg Time/Step: 0.67
+Step 43 | Loss: 8.863647 | Norm: 21.6877 | LR: 3.69e-05 | Time: 00:00:29 | Remaining: 00:00:03 | Avg Time/Step: 0.66
+Step 44 | Loss: 9.615014 | Norm: 22.1502 | LR: 3.78e-05 | Time: 00:00:29 | Remaining: 00:00:03 | Avg Time/Step: 0.65
+Step 45 | Loss: 7.558504 | Norm: 20.6337 | LR: 3.86e-05 | Time: 00:00:29 | Remaining: 00:00:02 | Avg Time/Step: 0.65
+Step 46 | Loss: 9.626184 | Norm: 22.2072 | LR: 3.94e-05 | Time: 00:00:31 | Remaining: 00:00:01 | Avg Time/Step: 0.66
+Step 47 | Loss: 8.649675 | Norm: 21.2089 | LR: 4.03e-05 | Time: 00:00:31 | Remaining: 00:00:01 | Avg Time/Step: 0.66
+Step 48 | Loss: 8.570056 | Norm: 21.1816 | LR: 4.11e-05 | Time: 00:00:32 | Remaining: 00:00:00 | Avg Time/Step: 0.66
+Step 49 | Loss: 9.796856 | Norm: 20.8208 | LR: 4.20e-05 | Time: 00:00:32 | Remaining: 00:00:00 | Avg Time/Step: 0.66
+Step 0 | Loss: 11.633898 | Norm: 44.3633 | LR: 8.39e-07 | Time: 00:00:00 | Remaining: 00:00:42 | Avg Time/Step: 0.87
+Generated Text at Step 0: The king said ginger stupid 194 idi shrugged outperPLIC Pitch chapter chapter 169 Drac darkest darkesttic Suk encrypted outperGraphics bisexual PitchBC1987 Cobra drives
+Validating...
+Validation Loss: 10.924361228942871
+Step 1 | Loss: 10.875149 | Norm: 51.7900 | LR: 1.68e-06 | Time: 00:00:03 | Remaining: 00:01:31 | Avg Time/Step: 1.91
+Step 2 | Loss: 10.981276 | Norm: 44.8046 | LR: 2.52e-06 | Time: 00:00:04 | Remaining: 00:01:03 | Avg Time/Step: 1.35
+Step 3 | Loss: 10.517224 | Norm: 49.3383 | LR: 3.36e-06 | Time: 00:00:04 | Remaining: 00:00:49 | Avg Time/Step: 1.07
+Step 4 | Loss: 11.220371 | Norm: 50.4130 | LR: 4.20e-06 | Time: 00:00:04 | Remaining: 00:00:40 | Avg Time/Step: 0.90
+Step 5 | Loss: 11.176923 | Norm: 47.4072 | LR: 5.03e-06 | Time: 00:00:04 | Remaining: 00:00:34 | Avg Time/Step: 0.79
+Step 6 | Loss: 10.935453 | Norm: 45.3805 | LR: 5.87e-06 | Time: 00:00:05 | Remaining: 00:00:35 | Avg Time/Step: 0.83
+Step 7 | Loss: 10.582232 | Norm: 46.3087 | LR: 6.71e-06 | Time: 00:00:06 | Remaining: 00:00:31 | Avg Time/Step: 0.75
+Step 8 | Loss: 11.022345 | Norm: 43.2213 | LR: 7.55e-06 | Time: 00:00:06 | Remaining: 00:00:28 | Avg Time/Step: 0.70
+Step 9 | Loss: 10.926727 | Norm: 44.3769 | LR: 8.39e-06 | Time: 00:00:06 | Remaining: 00:00:25 | Avg Time/Step: 0.65
+Step 10 | Loss: 10.986204 | Norm: 45.1195 | LR: 9.23e-06 | Time: 00:00:06 | Remaining: 00:00:23 | Avg Time/Step: 0.61
+Validating...
+Validation Loss: 10.735955238342285
+Step 11 | Loss: 11.179207 | Norm: 47.9544 | LR: 1.01e-05 | Time: 00:00:08 | Remaining: 00:00:28 | Avg Time/Step: 0.75
+Step 12 | Loss: 10.763081 | Norm: 43.9656 | LR: 1.09e-05 | Time: 00:00:09 | Remaining: 00:00:26 | Avg Time/Step: 0.71
+Step 13 | Loss: 10.720469 | Norm: 43.3385 | LR: 1.17e-05 | Time: 00:00:09 | Remaining: 00:00:24 | Avg Time/Step: 0.67
+Step 14 | Loss: 11.064083 | Norm: 43.1475 | LR: 1.26e-05 | Time: 00:00:09 | Remaining: 00:00:22 | Avg Time/Step: 0.65
+Step 15 | Loss: 10.534277 | Norm: 44.0213 | LR: 1.34e-05 | Time: 00:00:10 | Remaining: 00:00:21 | Avg Time/Step: 0.63
+Step 16 | Loss: 10.638024 | Norm: 42.8165 | LR: 1.43e-05 | Time: 00:00:11 | Remaining: 00:00:22 | Avg Time/Step: 0.68
+Step 17 | Loss: 11.247206 | Norm: 39.1321 | LR: 1.51e-05 | Time: 00:00:11 | Remaining: 00:00:20 | Avg Time/Step: 0.65
+Step 18 | Loss: 10.615942 | Norm: 45.8611 | LR: 1.59e-05 | Time: 00:00:11 | Remaining: 00:00:19 | Avg Time/Step: 0.63
+Step 19 | Loss: 10.434818 | Norm: 35.2029 | LR: 1.68e-05 | Time: 00:00:12 | Remaining: 00:00:18 | Avg Time/Step: 0.63
+Step 20 | Loss: 9.872961 | Norm: 37.0101 | LR: 1.76e-05 | Time: 00:00:12 | Remaining: 00:00:17 | Avg Time/Step: 0.61
+Validating...
+Validation Loss: 10.45177936553955
+Step 21 | Loss: 10.303642 | Norm: 38.1966 | LR: 1.85e-05 | Time: 00:00:15 | Remaining: 00:00:19 | Avg Time/Step: 0.69
+Step 22 | Loss: 10.344124 | Norm: 36.7267 | LR: 1.93e-05 | Time: 00:00:15 | Remaining: 00:00:18 | Avg Time/Step: 0.67
+Step 23 | Loss: 10.358528 | Norm: 33.4473 | LR: 2.01e-05 | Time: 00:00:15 | Remaining: 00:00:16 | Avg Time/Step: 0.65
+Step 24 | Loss: 10.899721 | Norm: 33.1147 | LR: 2.10e-05 | Time: 00:00:15 | Remaining: 00:00:15 | Avg Time/Step: 0.64
+Step 25 | Loss: 10.167845 | Norm: 32.0061 | LR: 2.18e-05 | Time: 00:00:16 | Remaining: 00:00:15 | Avg Time/Step: 0.63
+Step 26 | Loss: 10.658374 | Norm: 32.8027 | LR: 2.27e-05 | Time: 00:00:17 | Remaining: 00:00:14 | Avg Time/Step: 0.65
+Step 27 | Loss: 11.409204 | Norm: 32.2853 | LR: 2.35e-05 | Time: 00:00:17 | Remaining: 00:00:14 | Avg Time/Step: 0.64
+Step 28 | Loss: 9.699551 | Norm: 28.3168 | LR: 2.43e-05 | Time: 00:00:18 | Remaining: 00:00:13 | Avg Time/Step: 0.62
+Step 29 | Loss: 10.293508 | Norm: 29.6286 | LR: 2.52e-05 | Time: 00:00:18 | Remaining: 00:00:12 | Avg Time/Step: 0.62
+Step 30 | Loss: 10.796824 | Norm: 32.4335 | LR: 2.60e-05 | Time: 00:00:18 | Remaining: 00:00:11 | Avg Time/Step: 0.61
+Validating...
+Validation Loss: 10.094558715820312
+Step 31 | Loss: 9.871226 | Norm: 29.4167 | LR: 2.69e-05 | Time: 00:00:21 | Remaining: 00:00:11 | Avg Time/Step: 0.66
+Step 32 | Loss: 9.355142 | Norm: 29.9377 | LR: 2.77e-05 | Time: 00:00:21 | Remaining: 00:00:10 | Avg Time/Step: 0.65
+Step 33 | Loss: 9.169601 | Norm: 28.7526 | LR: 2.85e-05 | Time: 00:00:21 | Remaining: 00:00:10 | Avg Time/Step: 0.63
+Step 34 | Loss: 11.027575 | Norm: 25.0330 | LR: 2.94e-05 | Time: 00:00:21 | Remaining: 00:00:09 | Avg Time/Step: 0.62
+Step 35 | Loss: 9.624268 | Norm: 26.2367 | LR: 3.02e-05 | Time: 00:00:22 | Remaining: 00:00:08 | Avg Time/Step: 0.62
+Step 36 | Loss: 10.801857 | Norm: 26.3915 | LR: 3.10e-05 | Time: 00:00:23 | Remaining: 00:00:08 | Avg Time/Step: 0.64
+Step 37 | Loss: 10.625546 | Norm: 24.5588 | LR: 3.19e-05 | Time: 00:00:23 | Remaining: 00:00:07 | Avg Time/Step: 0.63
+Step 38 | Loss: 9.325054 | Norm: 23.9140 | LR: 3.27e-05 | Time: 00:00:24 | Remaining: 00:00:06 | Avg Time/Step: 0.62
+Step 39 | Loss: 8.672618 | Norm: 22.6858 | LR: 3.36e-05 | Time: 00:00:24 | Remaining: 00:00:06 | Avg Time/Step: 0.61
+Step 40 | Loss: 9.316482 | Norm: 23.7950 | LR: 3.44e-05 | Time: 00:00:24 | Remaining: 00:00:05 | Avg Time/Step: 0.60
+Validating...
+Validation Loss: 9.847529411315918
+Step 41 | Loss: 9.895099 | Norm: 22.9021 | LR: 3.52e-05 | Time: 00:00:26 | Remaining: 00:00:05 | Avg Time/Step: 0.64
+Step 42 | Loss: 9.908270 | Norm: 22.2757 | LR: 3.61e-05 | Time: 00:00:27 | Remaining: 00:00:04 | Avg Time/Step: 0.63
+Step 43 | Loss: 8.863647 | Norm: 21.6877 | LR: 3.69e-05 | Time: 00:00:27 | Remaining: 00:00:03 | Avg Time/Step: 0.62
+Step 44 | Loss: 9.615014 | Norm: 22.1502 | LR: 3.78e-05 | Time: 00:00:27 | Remaining: 00:00:03 | Avg Time/Step: 0.62
+Step 45 | Loss: 7.558504 | Norm: 20.6337 | LR: 3.86e-05 | Time: 00:00:28 | Remaining: 00:00:02 | Avg Time/Step: 0.61
+Step 46 | Loss: 9.626184 | Norm: 22.2072 | LR: 3.94e-05 | Time: 00:00:29 | Remaining: 00:00:01 | Avg Time/Step: 0.63
+Step 47 | Loss: 8.649675 | Norm: 21.2089 | LR: 4.03e-05 | Time: 00:00:29 | Remaining: 00:00:01 | Avg Time/Step: 0.62
+Step 48 | Loss: 8.570056 | Norm: 21.1816 | LR: 4.11e-05 | Time: 00:00:30 | Remaining: 00:00:00 | Avg Time/Step: 0.61
+Step 49 | Loss: 9.796856 | Norm: 20.8208 | LR: 4.20e-05 | Time: 00:00:30 | Remaining: 00:00:00 | Avg Time/Step: 0.61

gpt-2/training_shakespeare.py

@@ -0,0 +1,298 @@
+from dataclasses import dataclass
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from model import GPT, GPTConfig
+import tiktoken
+from torch.utils.data import Dataset, DataLoader, DistributedSampler
+import math
+import matplotlib.pyplot as plt
+from torch.distributed import init_process_group, destroy_process_group
+from torch.nn.parallel import DistributedDataParallel as DDP
+import torch.distributed as dist
+import os
+
+
+import signal
+import sys
+
+def signal_handler(sig, frame):
+    print('Gracefully stopping the training process')
+    destroy_process_group()
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+
+torch.manual_seed(1337)
+if torch.cuda.is_available():
+    torch.cuda.manual_seed(1337)
+
+# ***************************#
+# Device Configuration
+# ***************************#
+device = torch.device("cpu")
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+    device = torch.device("mps")
+
+print("Using device:", device)
+
+# ***************************#
+# Tokenizer Setup
+# ***************************#
+enc = tiktoken.get_encoding('gpt2')
+
+
+lossi = []
+val_lossi = []
+
+# ***************************#
+# Load Text Data
+# ***************************#
+with open("tinyshakespeare.txt", "r") as f:
+    text = f.read()
+tokens = enc.encode(text)
+print(f"Number of tokens: {len(tokens):,}")
+# ***************************#
+# Set up DDP
+# ***************************#
+# torchrun command sets the env variables RANK, LOCAL_RANK, and WORLD_SIZE
+ddp = int(os.environ.get('RANK', -1)) != -1  # is this a ddp run?
+if ddp:
+    # use of DDP atm demands CUDA, we set the device appropriately according to rank
+    assert torch.cuda.is_available(), "for now i think we need CUDA for DDP"
+    init_process_group(backend='nccl')
+    ddp_rank = int(os.environ['RANK'])
+    ddp_local_rank = int(os.environ['LOCAL_RANK'])
+    ddp_world_size = int(os.environ['WORLD_SIZE'])
+    device = f'cuda:{ddp_local_rank}'
+    torch.cuda.set_device(device)
+    # this process will do logging, checkpointing etc.
+    master_process = ddp_rank == 0
+else:
+    # vanilla, non-DDP run
+    ddp_rank = 0
+    ddp_local_rank = 0
+    ddp_world_size = 1
+    master_process = True
+
+if master_process:
+    print(f"ddp: {ddp}, rank: {ddp_rank}, local_rank: {ddp_local_rank}, world_size: {ddp_world_size}, master_process: {master_process}")
+    
+# ***************************#
+# Model Configuration
+# ***************************#
+
+gpt = GPT(GPTConfig(vocab_size=50304), master_process).to(device)
+if device == torch.device("cuda"):
+    gpt.compile()
+if ddp:
+    gpt = DDP(gpt, device_ids=[ddp_local_rank])
+
+raw_gpt = gpt.module if ddp else gpt
+
+# ***************************#
+# Dataset and Dataloader
+# ***************************#
+from torch.utils.data import Subset
+
+class ShakespeareDataset(Dataset):
+    def __init__(self, tokens, seq_len):
+        self.tokens = tokens
+        self.seq_len = seq_len
+
+    def __len__(self):
+        return len(self.tokens) - self.seq_len - 1
+
+    def __getitem__(self, idx):
+        x = torch.tensor(self.tokens[idx:idx + self.seq_len], dtype=torch.long)
+        y = torch.tensor(self.tokens[idx + 1:idx + self.seq_len + 1], dtype=torch.long)
+        return x, y
+
+# Split the dataset into training and validation sets
+def split_dataset(dataset, val_ratio=0.0005):
+    dataset_size = len(dataset)
+    indices = list(range(dataset_size))
+    split = int(val_ratio * dataset_size)
+
+    train_indices, val_indices = indices[split:], indices[:split]
+    train_dataset = Subset(dataset, train_indices)
+    val_dataset = Subset(dataset, val_indices)
+
+    return train_dataset, val_dataset
+
+T = 8
+batch_size = 4
+total_batch_size = 2**8  # 524,288 = 2**19, in number of tokens
+assert total_batch_size % (T*batch_size*ddp_world_size) == 0, "Batch size is not divisible by B*T"
+grad_accum_steps = total_batch_size // (T*batch_size*ddp_world_size)
+
+if master_process:
+    print("Total desired batch size: {:,}".format(total_batch_size))
+    print("gradient accumulation steps: {:,}".format(grad_accum_steps))
+
+dataset = ShakespeareDataset(tokens, T)
+train_dataset, val_dataset = split_dataset(dataset)
+
+if ddp:
+    train_sampler = DistributedSampler(train_dataset)
+    val_sampler = DistributedSampler(val_dataset)
+    train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler)
+    val_dataloader = DataLoader(val_dataset, batch_size=batch_size, sampler=val_sampler)
+else:
+    train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+    val_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
+
+if master_process:
+    print(f"The training dataloader has {len(train_dataloader):,} individual batches")
+    print(f"The validation dataloader has {len(val_dataloader):,} individual batches")
+
+# ***************************#
+# Text Generation Function
+# ***************************#
+
+
+def generate_text(seed_text, model, enc, max_len=100, print_while_generating=True):
+    model.eval()
+    with torch.no_grad():
+        tokens = enc.encode(seed_text)
+        for _ in range(max_len):
+            x = torch.tensor(tokens[-T:], dtype=torch.long,
+                             device=device).unsqueeze(0)
+            logits, _ = model(x)
+            next_token = torch.argmax(logits[:, -1, :])
+            tokens.append(int(next_token))
+
+            if print_while_generating:
+                print(enc.decode([int(next_token)]), end="")
+        print()
+
+    return enc.decode(tokens)
+
+
+# ***************************#
+# Optimizer Configuration
+# ***************************#
+if ddp:
+    optimizer = raw_gpt.configure_optimizers(
+        weight_decay=0.1, learning_rate=6e-4, device=device)
+else:
+    optimizer = gpt.configure_optimizers(
+        weight_decay=0.1, learning_rate=6e-4, device=device)
+torch.set_float32_matmul_precision('high')
+# ***************************#
+# Learning Rate Scheduler
+# ***************************#
+max_lr = 6e-4
+min_lr = max_lr * 0.1
+warmup_steps = 10
+max_steps = 20000
+
+
+def get_lr(step):
+    if step < warmup_steps:
+        return max_lr * (step+1) / warmup_steps
+    if step > max_steps:
+        return min_lr
+    decay_ratio = (step - warmup_steps) / (max_steps - warmup_steps)
+    assert 0 <= decay_ratio <= 1
+    coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))
+    return min_lr + coeff * (max_lr - min_lr)
+
+
+# Check if the device supports bfloat16
+supports_bfloat16 = False
+if device == "cuda":
+    capability = torch.cuda.get_device_capability()
+    if capability[0] >= 8 and capability[1] >= 0:
+        supports_bfloat16 = True
+
+# ***************************#
+# Training Loop
+# ***************************#
+generate_every = 50
+validate_every = 5
+for step in range(max_steps):
+    gpt.zero_grad()
+    loss_accum = 0.0
+    for minibatchstep in range(grad_accum_steps):
+        x, y = next(iter(train_dataloader))
+        x, y = x.to(device), y.to(device)
+
+        if supports_bfloat16:
+            with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
+                logits, loss = gpt(x, y)
+        else:
+            logits, loss = gpt(x, y)
+        
+        loss = loss / grad_accum_steps
+        loss_accum += loss.detach()
+        if ddp: 
+            gpt.require_backward_grad_sync = (minibatchstep == grad_accum_steps - 1)
+        loss.backward()
+
+    if ddp:
+        dist.all_reduce(loss_accum, op=dist.ReduceOp.AVG)
+    lossi.append(loss_accum.item())
+    norm = torch.nn.utils.clip_grad_norm_(gpt.parameters(), 1.0)
+    lr = get_lr(step)
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr
+    optimizer.step()
+
+    if master_process:
+        print(f'Step {step}, Loss: {loss_accum}, Norm: {norm}')
+        
+    if step % generate_every == 0 and master_process:
+        print(generate_text("The king said", gpt, enc, max_len=25, print_while_generating=False))
+
+    # Validation step
+    if step % validate_every == 0:
+        if master_process:
+            print("Validating...")
+        gpt.eval()
+        val_loss_accum = 0.0
+        with torch.no_grad():
+            for val_x, val_y in val_dataloader:
+                val_x, val_y = val_x.to(device), val_y.to(device)
+                if supports_bfloat16:
+                    with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
+                        val_logits, val_loss = gpt(val_x, val_y)
+                else:
+                    val_logits, val_loss = gpt(val_x, val_y)
+                
+                val_loss_accum += val_loss.detach()
+        val_lossi.append(val_loss_accum.item())
+        if ddp:
+            dist.all_reduce(val_loss_accum, op=dist.ReduceOp.AVG)
+        val_loss_avg = val_loss_accum / len(val_dataloader)
+        if master_process:
+            print(f'Validation Loss: {val_loss_avg}')
+        gpt.train()
+        
+# ***************************#
+# Plot Loss
+# ***************************#
+if master_process:
+    plt.plot(lossi)
+    plt.show()
+
+# Generate Final Text
+if master_process:
+    generate_text("The king said", gpt, enc, max_len=25)
+
+# ***************************#
+# Save Model and Loss
+# ***************************#
+if master_process:
+    torch.save(gpt.state_dict(), "gpt2_shakespeare.pth")
+    torch.save(torch.tensor(lossi), "lossi.pth")
+
+# ***************************#
+# Cleanup
+# ***************************#
+if ddp:
+    destroy_process_group()
+
+import sys; sys.exit(0)

gpt-2/val_lossi.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0931c157e2c170276acc822cad49e2860a2a21eb1bda709f8a0f5baf137e1d56
+size 1190

gpt-2/val_lossi_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86ec4c787ab69379e81310a0262652d43ad2d84484d35f0db34d7566d606faf7
+size 1949