Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:160878cb597e144209b89ea35c275fb6313098522417b36afbfcf424be5de609
+oid sha256:904c4cb7219efa4ac395305c3ab94378126d290c0afc862ef3a90774d56be16b
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:087813ab697aadc55c717501615cb2d868f4f078c5d63952114965454aa8268d
+oid sha256:ccb3bedd591dc05adcc366ccb7419995e39128f07f0e07d7592cd7f187a5c7fd
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:199fce353b16b3ea7a004aba16e650c3a24c6dcb7f9e2b152373e944b3f03425
+oid sha256:f8698665ad1893c3d7d8f0bbd607b0cebc6107a39cd0e83010448ab7af8d220e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49a531e722bef6990e0f2487405157db96b8b8fb14ae0e9a407dadecabc5f53d
+oid sha256:fba657d32ca917aa7eec719caf28d12b084905654f9bc9fe5fea5d325abb2cbf
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.08015908086610694,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=9-learning_rate=5.9e-06-warmup_ratio=0.070-min_lr=0.000-factor=0.540/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.518338488731772,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=10-learning_rate=7.6e-05-warmup_ratio=0.040-min_lr=0.000-factor=0.940/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 1.1627027988433838,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 2.1727,
+      "grad_norm": 0.7516372799873352,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.8486,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 1.1514556407928467,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.8457,
+      "grad_norm": 0.3815319538116455,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.1503,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 1.169623851776123,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.5884,
+      "grad_norm": 0.22006957232952118,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0563,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.8743982315063477,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.4466,
+      "grad_norm": 0.19324058294296265,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0313,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.9265111088752747,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.3545,
+      "grad_norm": 0.1767522692680359,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0206,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.7506417632102966,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.2917,
+      "grad_norm": 0.20005017518997192,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0154,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.5906476974487305,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.2453,
+      "grad_norm": 0.14517371356487274,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0121,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.676028847694397,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.2128,
+      "grad_norm": 0.1346338540315628,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0113,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.5698741674423218,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.1871,
+      "grad_norm": 0.13754339516162872,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0089,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.5416367650032043,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.1676,
+      "grad_norm": 0.14780430495738983,
+      "learning_rate": 7.582125965096081e-05,
+      "loss": 0.0078,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0023862129916040653,
-      "eval_e3_equal": 0.3105612019443217,
+      "eval_all_ligands_equal": 0.518338488731772,
+      "eval_e3_equal": 0.8000883782589483,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9965532479010164,
-      "eval_e3_heavy_atoms_difference": -0.9543968183826779,
-      "eval_e3_heavy_atoms_difference_norm": -0.05584347744651639,
+      "eval_e3_has_attachment_point(s)": 0.9925762262483429,
+      "eval_e3_heavy_atoms_difference": 0.48475475033141846,
+      "eval_e3_heavy_atoms_difference_norm": 0.011724540067924432,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9965532479010164,
-      "eval_has_all_attachment_points": 0.9602297834732656,
-      "eval_has_three_substructures": 0.9969067609368095,
-      "eval_heavy_atoms_difference": 16.153336279275297,
-      "eval_heavy_atoms_difference_norm": 0.20615734486032428,
-      "eval_linker_equal": 0.012019443216968627,
+      "eval_e3_valid": 0.9925762262483429,
+      "eval_has_all_attachment_points": 0.9901900132567388,
+      "eval_has_three_substructures": 0.9992929739284137,
+      "eval_heavy_atoms_difference": 4.0311091471498015,
+      "eval_heavy_atoms_difference_norm": 0.05126412535753748,
+      "eval_linker_equal": 0.7454706142288997,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9917808219178083,
-      "eval_linker_heavy_atoms_difference": 3.9414935925762262,
-      "eval_linker_heavy_atoms_difference_norm": 0.14213508720580156,
+      "eval_linker_has_attachment_point(s)": 0.99584622182943,
+      "eval_linker_heavy_atoms_difference": 0.3673000441891295,
+      "eval_linker_heavy_atoms_difference_norm": -0.001600385700560306,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9917808219178083,
-      "eval_loss": 0.35252776741981506,
-      "eval_num_fragments": 2.997083517454706,
-      "eval_poi_equal": 0.3250552364118427,
+      "eval_linker_valid": 0.99584622182943,
+      "eval_loss": 0.3114722669124603,
+      "eval_num_fragments": 2.999646486964207,
+      "eval_poi_equal": 0.7483870967741936,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.8409191338930623,
-      "eval_poi_heavy_atoms_difference": 5.727087936367654,
-      "eval_poi_heavy_atoms_difference_norm": 0.15969077298536322,
+      "eval_poi_has_attachment_point(s)": 0.9661511268228016,
+      "eval_poi_heavy_atoms_difference": 0.9739284136102518,
+      "eval_poi_heavy_atoms_difference_norm": 0.02216572405609002,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.8409191338930623,
-      "eval_reassembly": 0.0023862129916040653,
-      "eval_reassembly_nostereo": 0.003358373840035351,
-      "eval_runtime": 1524.6884,
-      "eval_samples_per_second": 7.421,
-      "eval_steps_per_second": 0.116,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.8365002209456474,
-      "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.5359856486320496,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.1511,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.47309595346450806,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.1374,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.43925419449806213,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.126,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.4663945436477661,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.116,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.4891926944255829,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.107,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.019619973486522314,
-      "eval_e3_equal": 0.45912505523641184,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9980556783031375,
-      "eval_e3_heavy_atoms_difference": -0.2529385771100309,
-      "eval_e3_heavy_atoms_difference_norm": -0.022332054888424536,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9980556783031375,
-      "eval_has_all_attachment_points": 0.9810870525850641,
-      "eval_has_three_substructures": 0.9985859478568272,
-      "eval_heavy_atoms_difference": 8.913212549712771,
-      "eval_heavy_atoms_difference_norm": 0.11521901930566276,
-      "eval_linker_equal": 0.05019885108263367,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9961113566062748,
-      "eval_linker_heavy_atoms_difference": 2.3817057003977022,
-      "eval_linker_heavy_atoms_difference_norm": 0.07995994133117267,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9961113566062748,
-      "eval_loss": 0.32926952838897705,
-      "eval_num_fragments": 2.99946973044631,
-      "eval_poi_equal": 0.48572691117984973,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9186036235086169,
-      "eval_poi_heavy_atoms_difference": 2.8219178082191783,
-      "eval_poi_heavy_atoms_difference_norm": 0.07881083658584855,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9186036235086169,
-      "eval_reassembly": 0.020150243040212106,
-      "eval_reassembly_nostereo": 0.025806451612903226,
-      "eval_runtime": 1537.7319,
+      "eval_poi_valid": 0.9661511268228016,
+      "eval_reassembly": 0.5254971277065842,
+      "eval_reassembly_nostereo": 0.5520989836500221,
+      "eval_runtime": 1537.7347,
       "eval_samples_per_second": 7.358,
       "eval_steps_per_second": 0.115,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9159522757401679,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.4543463885784149,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.1007,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.3903944492340088,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.0924,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.4319060742855072,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.0845,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.3749650716781616,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.078,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.4189089238643646,
-      "learning_rate": 5.875163053649505e-06,
-      "loss": 0.0722,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.08015908086610694,
-      "eval_e3_equal": 0.5646486964206805,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9987627043747238,
-      "eval_e3_heavy_atoms_difference": -0.524259832081308,
-      "eval_e3_heavy_atoms_difference_norm": -0.03533191702633512,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9987627043747238,
-      "eval_has_all_attachment_points": 0.9898365002209456,
-      "eval_has_three_substructures": 0.9993813521873619,
-      "eval_heavy_atoms_difference": 4.513389306230668,
-      "eval_heavy_atoms_difference_norm": 0.05765869939593611,
-      "eval_linker_equal": 0.15351303579319486,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9974370304904994,
-      "eval_linker_heavy_atoms_difference": 0.5383119752540875,
-      "eval_linker_heavy_atoms_difference_norm": -0.029146778634363005,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9974370304904994,
-      "eval_loss": 0.3140394687652588,
-      "eval_num_fragments": 2.9999116217410515,
-      "eval_poi_equal": 0.5788775961113566,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9430844012372956,
-      "eval_poi_heavy_atoms_difference": 1.6765355722492268,
-      "eval_poi_heavy_atoms_difference_norm": 0.045865474002063256,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9430844012372956,
-      "eval_reassembly": 0.08130799823243483,
-      "eval_reassembly_nostereo": 0.09889527176314626,
-      "eval_runtime": 1522.9676,
-      "eval_samples_per_second": 7.43,
-      "eval_steps_per_second": 0.116,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9407865665046399,
-      "step": 10000
+      "eval_valid": 0.9566062748563853,
+      "step": 5000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=9-learning_rate=5.9e-06-warmup_ratio=0.070-min_lr=0.000-factor=0.540",
+  "trial_name": "trial-number=10-learning_rate=7.6e-05-warmup_ratio=0.040-min_lr=0.000-factor=0.940",
   "trial_params": {
-    "factor": 0.54,
-    "learning_rate": 5.875163053649505e-06,
-    "min_lr": 2.2429164319904815e-11,
-    "warmup_ratio": 0.06999999999999999
+    "factor": 0.94,
+    "learning_rate": 7.582125965096081e-05,
+    "min_lr": 5.001256479531022e-11,
+    "warmup_ratio": 0.04
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1dd83e329f1b1df7007c5d8a913275794ba72cfa3b5bd143db424609f8dd1cd
+oid sha256:b141933d3d59f747dba8d7362622a470e9dacf61df786a6eff32bf95c9982720
 size 7544