Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9699af8b31d5fe42c1af26c9faf85f24753409b2a06ef6cfd48bb808bcea6236
+oid sha256:5953bc37830c0c9e130eddd97b29a56d7ce7d70cb737598959a31ac6d3a59c3a
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e79e6e5d5729273468192b4fe4a270ecee11270d5934d08e69dcb017fb0cf61
+oid sha256:e5df57c13de7cc8b3b1efdc3067d551ae3c6dd6c9a1ec5c54717c50102db40f8
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69b51f0a88f5a559dfad8b70382ccedcf3137fcef1d7f7521c81f6717533dfa9
+oid sha256:eaa2d2f6ccfb817ca63eb880a0c49fc48ccfe7f3d16359ac26bdeecdd99e8e3e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f7b2aea94214e16c346a0cd248ea8fde97c3e87f6418c297a784de8cd993010
+oid sha256:29f2ebac42d8b2630f1c35545f799d405a0d97159777518a22a7cbcd0cb16c6b
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.43809102960671675,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=8-learning_rate=1.3e-05-warmup_ratio=0.010-min_lr=0.000-factor=0.220/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0023862129916040653,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=9-learning_rate=5.9e-06-warmup_ratio=0.070-min_lr=0.000-factor=0.540/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.9919585585594177,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 1.6025,
+      "grad_norm": 1.1627027988433838,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 2.1727,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.9148474931716919,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.536,
+      "grad_norm": 1.1514556407928467,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.8457,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.7302022576332092,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.3244,
+      "grad_norm": 1.169623851776123,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.5884,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.6461985111236572,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.2247,
+      "grad_norm": 0.8743982315063477,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.4466,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.5478264689445496,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.1687,
+      "grad_norm": 0.9265111088752747,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.3545,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.46242278814315796,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.1336,
+      "grad_norm": 0.7506417632102966,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.2917,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.36126068234443665,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.1064,
+      "grad_norm": 0.5906476974487305,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.2453,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.3201972246170044,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0862,
+      "grad_norm": 0.676028847694397,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.2128,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.35753563046455383,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0704,
+      "grad_norm": 0.5698741674423218,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.1871,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.3595835566520691,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0591,
+      "grad_norm": 0.5416367650032043,
+      "learning_rate": 5.875163053649505e-06,
+      "loss": 0.1676,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.158285461776403,
-      "eval_e3_equal": 0.6709677419354839,
+      "eval_all_ligands_equal": 0.0023862129916040653,
+      "eval_e3_equal": 0.3105612019443217,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9984091913389306,
-      "eval_e3_heavy_atoms_difference": -0.1937251436146708,
-      "eval_e3_heavy_atoms_difference_norm": -0.022125648951058074,
+      "eval_e3_has_attachment_point(s)": 0.9965532479010164,
+      "eval_e3_heavy_atoms_difference": -0.9543968183826779,
+      "eval_e3_heavy_atoms_difference_norm": -0.05584347744651639,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9984091913389306,
-      "eval_has_all_attachment_points": 0.9885992045956694,
-      "eval_has_three_substructures": 0.9993813521873619,
-      "eval_heavy_atoms_difference": 6.95793194874061,
-      "eval_heavy_atoms_difference_norm": 0.08973210339619288,
-      "eval_linker_equal": 0.268935041979673,
+      "eval_e3_valid": 0.9965532479010164,
+      "eval_has_all_attachment_points": 0.9602297834732656,
+      "eval_has_three_substructures": 0.9969067609368095,
+      "eval_heavy_atoms_difference": 16.153336279275297,
+      "eval_heavy_atoms_difference_norm": 0.20615734486032428,
+      "eval_linker_equal": 0.012019443216968627,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9971718957136544,
-      "eval_linker_heavy_atoms_difference": 1.9495360141405214,
-      "eval_linker_heavy_atoms_difference_norm": 0.07550810263300518,
+      "eval_linker_has_attachment_point(s)": 0.9917808219178083,
+      "eval_linker_heavy_atoms_difference": 3.9414935925762262,
+      "eval_linker_heavy_atoms_difference_norm": 0.14213508720580156,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9971718957136544,
-      "eval_loss": 0.2988537549972534,
-      "eval_num_fragments": 2.999734865223155,
-      "eval_poi_equal": 0.6062748563853292,
+      "eval_linker_valid": 0.9917808219178083,
+      "eval_loss": 0.35252776741981506,
+      "eval_num_fragments": 2.997083517454706,
+      "eval_poi_equal": 0.3250552364118427,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9368095448519664,
-      "eval_poi_heavy_atoms_difference": 2.0343791427308884,
-      "eval_poi_heavy_atoms_difference_norm": 0.06010484602679077,
+      "eval_poi_has_attachment_point(s)": 0.8409191338930623,
+      "eval_poi_heavy_atoms_difference": 5.727087936367654,
+      "eval_poi_heavy_atoms_difference_norm": 0.15969077298536322,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9368095448519664,
-      "eval_reassembly": 0.16217410517012815,
-      "eval_reassembly_nostereo": 0.18382677861246133,
-      "eval_runtime": 1527.9721,
-      "eval_samples_per_second": 7.405,
+      "eval_poi_valid": 0.8409191338930623,
+      "eval_reassembly": 0.0023862129916040653,
+      "eval_reassembly_nostereo": 0.003358373840035351,
+      "eval_runtime": 1524.6884,
+      "eval_samples_per_second": 7.421,
       "eval_steps_per_second": 0.116,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9341581970835174,
+      "eval_valid": 0.8365002209456474,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.3542541563510895,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0505,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.26621213555336,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0437,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.2528485357761383,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0379,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.27762725949287415,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0338,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.21909886598587036,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0295,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.35271763146266016,
-      "eval_e3_equal": 0.7418471056120195,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9924878479893946,
-      "eval_e3_heavy_atoms_difference": 0.23252319929297394,
-      "eval_e3_heavy_atoms_difference_norm": -0.0017201594206716537,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9924878479893946,
-      "eval_has_all_attachment_points": 0.9860362350861688,
-      "eval_has_three_substructures": 0.9992929739284137,
-      "eval_heavy_atoms_difference": 5.737074679628812,
-      "eval_heavy_atoms_difference_norm": 0.07487054087890593,
-      "eval_linker_equal": 0.5281484754750332,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9971718957136544,
-      "eval_linker_heavy_atoms_difference": 0.5623508616880247,
-      "eval_linker_heavy_atoms_difference_norm": 0.0005939261504664401,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9971718957136544,
-      "eval_loss": 0.29675376415252686,
-      "eval_num_fragments": 3.0,
-      "eval_poi_equal": 0.6851082633672116,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9439681838267786,
-      "eval_poi_heavy_atoms_difference": 1.8425099425541316,
-      "eval_poi_heavy_atoms_difference_norm": 0.055108029319327526,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9439681838267786,
-      "eval_reassembly": 0.3578435704816615,
-      "eval_reassembly_nostereo": 0.3903667697746354,
-      "eval_runtime": 1506.3107,
-      "eval_samples_per_second": 7.512,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9358373840035351,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.223163440823555,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0263,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.2685829699039459,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.024,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.23083259165287018,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0214,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.2764021158218384,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0195,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.20302872359752655,
-      "learning_rate": 1.2789275019323142e-05,
-      "loss": 0.0179,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.43809102960671675,
-      "eval_e3_equal": 0.7737516570923553,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9943437914273089,
-      "eval_e3_heavy_atoms_difference": 0.12558550596553247,
-      "eval_e3_heavy_atoms_difference_norm": -0.004236009362451621,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9943437914273089,
-      "eval_has_all_attachment_points": 0.9902783915156872,
-      "eval_has_three_substructures": 0.9993813521873619,
-      "eval_heavy_atoms_difference": 5.090941228457799,
-      "eval_heavy_atoms_difference_norm": 0.06648832808261415,
-      "eval_linker_equal": 0.6404772425983208,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9971718957136544,
-      "eval_linker_heavy_atoms_difference": 0.6052143172779496,
-      "eval_linker_heavy_atoms_difference_norm": 0.013539501259176759,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9971718957136544,
-      "eval_loss": 0.30280253291130066,
-      "eval_num_fragments": 2.9999116217410515,
-      "eval_poi_equal": 0.7100309323906319,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9496243923994697,
-      "eval_poi_heavy_atoms_difference": 1.5444984533804684,
-      "eval_poi_heavy_atoms_difference_norm": 0.04371697247936184,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9496243923994697,
-      "eval_reassembly": 0.44498453380468406,
-      "eval_reassembly_nostereo": 0.47927529827662396,
-      "eval_runtime": 1498.7629,
-      "eval_samples_per_second": 7.55,
-      "eval_steps_per_second": 0.118,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.942907644719399,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=8-learning_rate=1.3e-05-warmup_ratio=0.010-min_lr=0.000-factor=0.220",
+  "trial_name": "trial-number=9-learning_rate=5.9e-06-warmup_ratio=0.070-min_lr=0.000-factor=0.540",
   "trial_params": {
-    "factor": 0.22,
-    "learning_rate": 1.2789275019323142e-05,
-    "min_lr": 3.966711717977572e-11,
-    "warmup_ratio": 0.01
+    "factor": 0.54,
+    "learning_rate": 5.875163053649505e-06,
+    "min_lr": 2.2429164319904815e-11,
+    "warmup_ratio": 0.06999999999999999
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7482535826fb3d396cd1599682322d70f83ade3cf9e3ac8479e85eef0257ec8a
+oid sha256:d1dd83e329f1b1df7007c5d8a913275794ba72cfa3b5bd143db424609f8dd1cd
 size 7544