global init lm

app.py

@@ -20,6 +20,9 @@ from transformers import (
 
 from methods import gdc_api_calls, utilities
 
+# global init to test guidance speed up
+lm = Transformers(model=model, tokenizer=tok)
+
 # set up various tokens
 hf_TOKEN = os.environ.get("hf_svc_ctds", False)
 
@@ -327,26 +330,26 @@ def construct_and_execute_api_call(query):
 def generate_percentage_response(modified_query):
     # set_seed(1042)
     regex = "The final response is: \d*\.\d*%"
-    lm = Transformers(model=model, tokenizer=tok)
-    lm += modified_query
-    lm += guidance_gen("pct_response", n=1, temperature=0, max_tokens=40, regex=regex)
-    return lm["pct_response"]
+    session = lm.copy()
+    session += modified_query
+    session += guidance_gen("pct_response", n=1, temperature=0, max_tokens=40, regex=regex)
+    return session["pct_response"]
 
 
 # generate llama model descriptive response
 @utilities.timeit
 @spaces.GPU(duration=10)
-def generate_descriptive_response(modified_query):  
-    lm = Transformers(model=model, tokenizer=tok)
-    lm += modified_query
-    lm += guidance_gen(
+def generate_descriptive_response(modified_query):
+    session = lm.copy()
+    session += modified_query
+    session += guidance_gen(
         "desc_response",
         n=1,
         temperature=0,
         max_tokens=100,
         regex="^[^\\n]*[.\S+]$",
     )
-    return lm["desc_response"]
+    return session["desc_response"]
 
 
 @utilities.timeit