speculative decoding initial impl completed (+6 squashed commit)

Squashed commit:

[0a6306ca0] draft wip dont use (will be squashed)

[a758a1c9c] wip dont use (will be squashed)

[e1994d3ce] wip dont use

[f59690d68] wip

[77228147d] wip on spec decoding. dont use yet

[2445bca54] wip adding speculative decoding (+1 squashed commits)

Squashed commits:

[50e341bb7] wip adding speculative decoding

otherarch/otherarch.h

@@ -507,4 +507,12 @@ struct llava_image
     float * clp_img_embd = nullptr; //this holds dynamic memory and must be freed each use!
 };
 
+struct speculative_draft_result
+{
+    std::vector<int32_t> draftids;
+    std::vector<float *> actual_logits;
+    bool draft_success = false;
+    int drafted_amount = 0;
+};
+
 const float default_norm_eps = 1e-5f;

otherarch/sdcpp/sdtype_adapter.cpp

@@ -357,11 +357,12 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
     int img2imgC = 3; // Assuming RGB image
     std::vector<uint8_t> resized_image_buf(img2imgW * img2imgH * img2imgC);
 
+    std::string ts = get_timestamp_str();
     if(!is_quiet)
     {
-        printf("\nGenerating Image (%d steps)\n",inputs.sample_steps);
+        printf("\n[%s] Generating Image (%d steps)\n",ts.c_str(),inputs.sample_steps);
     }else{
-        printf("\nGenerating (%d st.)\n",inputs.sample_steps);
+        printf("\n[%s] Generating (%d st.)\n",ts.c_str(),inputs.sample_steps);
     }
 
     fflush(stdout);

otherarch/utils.cpp

@@ -8,6 +8,7 @@
 #include <locale>
 #include <codecvt>
 #include <sstream>
+#include <ctime>
 
 
 void utreplace(std::string & str, const std::string & needle, const std::string & replacement) {
@@ -302,3 +303,14 @@ std::vector<uint8_t> kcpp_base64_decode(const std::string & encoded_string)
 
     return ret;
 }
+
+std::string get_timestamp_str()
+{
+    std::time_t t = std::time(nullptr);
+    std::tm* now = std::localtime(&t);
+    char buffer[16]; // Buffer to hold "hh:mm:ss" and null terminator
+    std::sprintf(buffer, "%02d:%02d:%02d", now->tm_hour, now->tm_min, now->tm_sec);
+    // Convert the buffer to a std::string
+    std::string timestamp(buffer);
+    return timestamp;
+}

otherarch/utils.h

@@ -57,3 +57,5 @@ bool should_transpose_layer(std::string name);
 void kcpp_graph_compute_helper(ggml_v3_cgraph * graph, int n_threads);
 
 std::vector<uint8_t> kcpp_base64_decode(const std::string & encoded_string);
+
+std::string get_timestamp_str();

otherarch/whispercpp/whisper_adapter.cpp

@@ -273,11 +273,12 @@ whisper_generation_outputs whispertype_generate(const whisper_generation_inputs
 
     // output text transcription
     whisper_output_text = output_txt(whisper_ctx, pcmf32s);
+    std::string ts = get_timestamp_str();
     if(!inputs.quiet)
     {
-        printf("\nWhisper Transcribe Output: %s",whisper_output_text.c_str());
+        printf("\n[%s] Whisper Transcribe Output: %s",ts.c_str(),whisper_output_text.c_str());
     } else {
-        printf("\nWhisper Transcribe Done.");
+        printf("\n[%s] Whisper Transcribe Done.",ts.c_str());
     }
     output.text = whisper_output_text.c_str();
     output.status = 1;