Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df1101b9ef48c8a8e13a86f96d7c87c77002e5901c7124cedc5f1c4abb0269a3
+oid sha256:8db8efcad1aad20ea13d8d7d9984c578dc26107113f2497cf58241ecf64f4d31
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0da88d6c1776a599023ee1ccfcd48faaa4e5554c05dc520a4e1ca8da964dadd1
+oid sha256:560f8225b4e422be5f485e5789c61967387596cfa41340c896b0266a600274ac
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e83fecf83dbd00bd37cc0cf9fceb9e941a3938dfcac00cdc765aed3a0abd49a0
+oid sha256:531dd6bbf77c44f3e63c47977249a318aeba5dfdb9fc88095e97b169b391cfae
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45c9aa4312086aa533a59b41f63cd1a73b3f25425cdd947baf7efebf9cc54e57
+oid sha256:7cfc9e052be169b4866b608e61d6ced510b87df870008f31ab0cca1e2c28c1a8
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.014228899690676094,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=4-learning_rate=4.2e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.800/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=5-learning_rate=1.1e-06-warmup_ratio=0.070-min_lr=0.000-factor=0.910/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.2561712265014648,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 2.4831,
+      "grad_norm": 3.6603024005889893,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 4.2002,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.2166751623153687,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 1.0063,
+      "grad_norm": 1.9637521505355835,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 1.9187,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.246788501739502,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.7292,
+      "grad_norm": 1.449631929397583,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 1.4159,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.0463918447494507,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.5705,
+      "grad_norm": 1.434281826019287,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 1.2066,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 1.0265284776687622,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.4664,
+      "grad_norm": 1.311953067779541,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 1.0605,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 1.0784770250320435,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.3926,
+      "grad_norm": 1.3984962701797485,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 0.9508,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.7869899272918701,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.3353,
+      "grad_norm": 1.29249906539917,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 0.8594,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 1.0476751327514648,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.2925,
+      "grad_norm": 1.4493184089660645,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 0.786,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.7107057571411133,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.2574,
+      "grad_norm": 1.42921781539917,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 0.7204,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.6933206915855408,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.2316,
+      "grad_norm": 1.4308568239212036,
+      "learning_rate": 1.1386440825931043e-06,
+      "loss": 0.6681,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 8.837825894829872e-05,
-      "eval_e3_equal": 0.19681838267786125,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.04348210340256297,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9943437914273089,
-      "eval_e3_heavy_atoms_difference": -1.1870967741935483,
-      "eval_e3_heavy_atoms_difference_norm": -0.068259312401578,
+      "eval_e3_has_attachment_point(s)": 0.9496243923994697,
+      "eval_e3_heavy_atoms_difference": 1.6608926204153778,
+      "eval_e3_heavy_atoms_difference_norm": 0.007137613196833208,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9943437914273089,
-      "eval_has_all_attachment_points": 0.940521431727795,
-      "eval_has_three_substructures": 0.9947856827220504,
-      "eval_heavy_atoms_difference": 24.551922227132124,
-      "eval_heavy_atoms_difference_norm": 0.31476917674862775,
-      "eval_linker_equal": 0.005125939019001326,
+      "eval_e3_valid": 0.9496243923994697,
+      "eval_has_all_attachment_points": 0.6837825894829872,
+      "eval_has_three_substructures": 0.950596553247901,
+      "eval_heavy_atoms_difference": 64.6774193548387,
+      "eval_heavy_atoms_difference_norm": 0.8393433147141454,
+      "eval_linker_equal": 0.0013256738842244808,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9897481219619974,
-      "eval_linker_heavy_atoms_difference": 5.2481661511268225,
-      "eval_linker_heavy_atoms_difference_norm": 0.21898110136881632,
+      "eval_linker_has_attachment_point(s)": 0.9260273972602739,
+      "eval_linker_heavy_atoms_difference": 9.002651347768449,
+      "eval_linker_heavy_atoms_difference_norm": 0.45586405473119984,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9897481219619974,
-      "eval_loss": 0.386488139629364,
-      "eval_num_fragments": 2.99531595227574,
-      "eval_poi_equal": 0.1725143614670791,
+      "eval_linker_valid": 0.9260273972602739,
+      "eval_loss": 0.7126381993293762,
+      "eval_num_fragments": 3.0372956252761822,
+      "eval_poi_equal": 0.00017675651789659743,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.748210340256297,
-      "eval_poi_heavy_atoms_difference": 9.451701281484755,
-      "eval_poi_heavy_atoms_difference_norm": 0.26897859855195805,
+      "eval_poi_has_attachment_point(s)": 0.29854175872735306,
+      "eval_poi_heavy_atoms_difference": 25.908882015024304,
+      "eval_poi_heavy_atoms_difference_norm": 0.7922565446145508,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.748210340256297,
-      "eval_reassembly": 0.00017675651789659743,
-      "eval_reassembly_nostereo": 0.00035351303579319487,
-      "eval_runtime": 1509.5004,
-      "eval_samples_per_second": 7.496,
-      "eval_steps_per_second": 0.117,
+      "eval_poi_valid": 0.29854175872735306,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 1368.1465,
+      "eval_samples_per_second": 8.27,
+      "eval_steps_per_second": 0.129,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.7445868316394167,
+      "eval_valid": 0.2896155545735749,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.5555406808853149,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.2088,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.5531201958656311,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1901,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.46529698371887207,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1745,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.538395881652832,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1613,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.4872206449508667,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1496,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.003358373840035351,
-      "eval_e3_equal": 0.34025629695095005,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9990278391515687,
-      "eval_e3_heavy_atoms_difference": -0.9244365885992046,
-      "eval_e3_heavy_atoms_difference_norm": -0.05036084634398262,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9990278391515687,
-      "eval_has_all_attachment_points": 0.9679186920017676,
-      "eval_has_three_substructures": 0.999116217410517,
-      "eval_heavy_atoms_difference": 13.730181175430843,
-      "eval_heavy_atoms_difference_norm": 0.17663834197618247,
-      "eval_linker_equal": 0.015554573574900574,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9950508174988952,
-      "eval_linker_heavy_atoms_difference": 3.765002209456474,
-      "eval_linker_heavy_atoms_difference_norm": 0.1332725666526942,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9950508174988952,
-      "eval_loss": 0.34545832872390747,
-      "eval_num_fragments": 2.9998232434821035,
-      "eval_poi_equal": 0.35704816615112683,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8675209898365002,
-      "eval_poi_heavy_atoms_difference": 4.575430844012373,
-      "eval_poi_heavy_atoms_difference_norm": 0.13147539573241804,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8675209898365002,
-      "eval_reassembly": 0.00344675209898365,
-      "eval_reassembly_nostereo": 0.004595669465311533,
-      "eval_runtime": 1597.4791,
-      "eval_samples_per_second": 7.083,
-      "eval_steps_per_second": 0.111,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8638091029606717,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.5376918911933899,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1402,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.3944121301174164,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1315,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.48968130350112915,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.1237,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.4314245879650116,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.117,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.4241904616355896,
-      "learning_rate": 4.170090767711237e-06,
-      "loss": 0.111,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.014228899690676094,
-      "eval_e3_equal": 0.43243482103402564,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9985859478568272,
-      "eval_e3_heavy_atoms_difference": -0.7708351745470614,
-      "eval_e3_heavy_atoms_difference_norm": -0.046181472855197955,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9985859478568272,
-      "eval_has_all_attachment_points": 0.9783473265576668,
-      "eval_has_three_substructures": 0.9987627043747238,
-      "eval_heavy_atoms_difference": 7.829960229783473,
-      "eval_heavy_atoms_difference_norm": 0.09936110916411747,
-      "eval_linker_equal": 0.040565620857269113,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9955810870525851,
-      "eval_linker_heavy_atoms_difference": 1.6191780821917807,
-      "eval_linker_heavy_atoms_difference_norm": 0.023629316247238026,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9955810870525851,
-      "eval_loss": 0.32917484641075134,
-      "eval_num_fragments": 2.9992929739284135,
-      "eval_poi_equal": 0.4608926204153778,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9138311975254088,
-      "eval_poi_heavy_atoms_difference": 2.668316394167035,
-      "eval_poi_heavy_atoms_difference_norm": 0.06637303713815954,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9138311975254088,
-      "eval_reassembly": 0.014670790985417587,
-      "eval_reassembly_nostereo": 0.018912947414935925,
-      "eval_runtime": 1533.4309,
-      "eval_samples_per_second": 7.379,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9113566062748564,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=4-learning_rate=4.2e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.800",
+  "trial_name": "trial-number=5-learning_rate=1.1e-06-warmup_ratio=0.070-min_lr=0.000-factor=0.910",
   "trial_params": {
-    "factor": 0.8,
-    "learning_rate": 4.170090767711237e-06,
-    "min_lr": 6.0724410942321634e-12,
-    "warmup_ratio": 0.03
+    "factor": 0.91,
+    "learning_rate": 1.1386440825931043e-06,
+    "min_lr": 1.1386082331680292e-12,
+    "warmup_ratio": 0.06999999999999999
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb3a31938db14c7e84935e0b50cb36c1f4b49136a4342b4b9c8e65d937342068
+oid sha256:a378ac23f30a31ba156b2d137739247698d58210a372732645c0744c38de224f
 size 7544