From 7ee359a59b8e472d9415ff760a9637797bff4b50 Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Tue, 20 Aug 2024 19:02:16 +0800
Subject: [PATCH] on multigpu setups, pick lowest free mem instead of highest
 for auto layers

---
 klite.embd   | 96 +++++++++++++++++++++++++++++++++++++++++++---------
 koboldcpp.py | 14 +++++---
 2 files changed, 89 insertions(+), 21 deletions(-)

diff --git a/klite.embd b/klite.embd
index c592fea81..aba6a1d2e 100644
--- a/klite.embd
+++ b/klite.embd
@@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
 -->
 
 <script>
-	const LITEVER = 165;
+	const LITEVER = 167;
 	const urlParams = new URLSearchParams(window.location.search);
 	const localflag = true;
 	const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@@ -734,6 +734,9 @@ Current version indicated by LITEVER below.
 		.color_gray {
 			color: #9b9b9b;
 		}
+		.color_lightgray {
+			color: #bbbbbb;
+		}
 		.color_red {
 			color: #ff7967;
 		}
@@ -4266,6 +4269,7 @@ Current version indicated by LITEVER below.
 		opmode: 4, //what mode are we in? 1=story, 2=adventure, 3=chat, 4=instruct
 		adventure_is_action: false, //in adventure mode, determine story or action
 		adventure_context_mod: true, //extra injection for adventure mode
+		fix_alpaca_leak: true, //prevents leaking when Alpaca instruct format is used on crappy models
 		chat_context_mod: true, //extra injection for chat mode
 		chatname: "User", //name to use in chat
 		chatopponent: defaultchatopponent,
@@ -4293,7 +4297,8 @@ Current version indicated by LITEVER below.
 		img_autogen: false,
 		img_allownsfw: true,
 		img_cfgscale: 7,
-		img_allowhd: false,
+		img_allowhd: true,
+		img_crop: false,
 		img_img2imgstr: 0.6,
 		img_steps: 20,
 		img_sampler: "Euler a",
@@ -9603,6 +9608,7 @@ Current version indicated by LITEVER below.
 		document.getElementById("inject_jailbreak_instruct").checked = localsettings.inject_jailbreak_instruct;
 		document.getElementById("idle_responses").value = localsettings.idle_responses;
 		document.getElementById("idle_duration").value = localsettings.idle_duration;
+		document.getElementById("fix_alpaca_leak").checked = localsettings.fix_alpaca_leak;
 		document.getElementById("adventure_context_mod").checked = localsettings.adventure_context_mod;
 		document.getElementById("chat_context_mod").checked = localsettings.chat_context_mod;
 		document.getElementById("instruct_has_markdown").checked = localsettings.instruct_has_markdown;
@@ -9674,6 +9680,7 @@ Current version indicated by LITEVER below.
 
 		document.getElementById("tokenstreammode").value = localsettings.tokenstreammode;
 		document.getElementById("img_allowhd").checked = localsettings.img_allowhd;
+		document.getElementById("img_crop").checked = localsettings.img_crop;
 		document.getElementById("img_autogen").checked = localsettings.img_autogen;
 		document.getElementById("save_images").checked = localsettings.save_images;
 		document.getElementById("save_remote_images").checked = localsettings.save_remote_images;
@@ -9915,6 +9922,7 @@ Current version indicated by LITEVER below.
 		localsettings.inject_jailbreak_instruct = (document.getElementById("inject_jailbreak_instruct").checked ? true : false);
 		localsettings.idle_responses = document.getElementById("idle_responses").value;
 		localsettings.idle_duration = document.getElementById("idle_duration").value;
+		localsettings.fix_alpaca_leak = (document.getElementById("fix_alpaca_leak").checked ? true : false);
 		localsettings.adventure_context_mod = (document.getElementById("adventure_context_mod").checked ? true : false);
 		localsettings.chat_context_mod = (document.getElementById("chat_context_mod").checked ? true : false);
 		localsettings.instruct_has_markdown = (document.getElementById("instruct_has_markdown").checked ? true : false);
@@ -9960,6 +9968,7 @@ Current version indicated by LITEVER below.
 		localsettings.image_negprompt = document.getElementById("negpromptinput").value;
 		localsettings.grammar = pendinggrammar;
 		localsettings.tokenstreammode = document.getElementById("tokenstreammode").value;
+		localsettings.img_crop = (document.getElementById("img_crop").checked ? true : false);
 		localsettings.img_allowhd = (document.getElementById("img_allowhd").checked ? true : false);
 		localsettings.img_autogen = (document.getElementById("img_autogen").checked ? true : false);
 		localsettings.save_images = (document.getElementById("save_images").checked ? true : false);
@@ -11965,6 +11974,10 @@ Current version indicated by LITEVER below.
 			let st = get_instruct_starttag(true);
 			let et = get_instruct_endtag(true);
 			seqs = [st, et];
+			if(localsettings.fix_alpaca_leak && st.toLowerCase().includes("### instruction"))
+			{
+				seqs.push("\n\n### ");
+			}
 			if(localsettings.inject_chatnames_instruct)
 			{
 				if(localsettings.chatname!="")
@@ -13611,7 +13624,7 @@ Current version indicated by LITEVER below.
 		}
 	}
 
-	function compressImage(inputDataUri, onDone, isJpeg=true, fixedSize=true, maxSize=NO_HD_RES_PX, quality = 0.35, forceCrop=false) {
+	function compressImage(inputDataUri, onDone, isJpeg=true, fixedSize=true, maxSize=NO_HD_RES_PX, quality = 0.35, forceAspect=false) {
 		let img = document.createElement('img');
 		let wantedWidth = maxSize;
 		let wantedHeight = maxSize;
@@ -13645,7 +13658,7 @@ Current version indicated by LITEVER below.
 			canvas.height = wantedHeight;
 
 			// We resize the image with the canvas method
-			if(forceCrop)
+			if(forceAspect)
 			{
 				let minsizeW = Math.min(origW, origH);
 				let minsizeH = Math.min(origW, origH);
@@ -13671,9 +13684,26 @@ Current version indicated by LITEVER below.
 					canvas.height = wantedHeight = maxSize;
 				}
 
-				let mx = (origW - minsizeW) / 2;
-				let my = (origH - minsizeH) / 2;
-				ctx.drawImage(this, mx, my, minsizeW, minsizeH, 0, 0, wantedWidth, wantedHeight);
+				let newWidth, newHeight, mx, my;
+				if (wantedWidth / wantedHeight > aspectratio) {
+					newHeight = wantedHeight;
+					newWidth = wantedHeight * aspectratio;
+				} else {
+					newWidth = wantedWidth;
+					newHeight = wantedWidth / aspectratio;
+				}
+
+				if (localsettings.img_crop) {
+					mx = (origW - minsizeW) / 2;
+					my = (origH - minsizeH) / 2;
+					ctx.drawImage(this, mx, my, minsizeW, minsizeH, 0, 0, wantedWidth, wantedHeight);
+				} else {
+					mx = (wantedWidth - newWidth) / 2;
+					my = (wantedHeight - newHeight) / 2;
+					ctx.fillStyle = "black";
+					ctx.fillRect(0, 0, wantedWidth, wantedHeight);
+					ctx.drawImage(this, mx, my, newWidth, newHeight);
+				}
 			}else{
 				ctx.drawImage(this, 0, 0, wantedWidth, wantedHeight);
 			}
@@ -14244,16 +14274,25 @@ Current version indicated by LITEVER below.
 			if (isSupported) {
 				warn_on_quit = true;
 				const selection = window.getSelection();
-				if(selection.focusNode!=null && selection.focusNode.parentElement!=null
-				&& selection.focusNode.parentElement.classList.contains("txtchunk"))
+				let foundparent = null;
+				if (selection.focusNode != null && selection.focusNode.parentElement != null
+					&& selection.focusNode.parentElement.classList.contains("txtchunk")) {
+					foundparent = selection.focusNode.parentElement;
+				} else if (selection.focusNode != null && selection.focusNode.parentElement != null
+					&& selection.focusNode.parentElement.parentElement != null
+					&& selection.focusNode.parentElement.parentElement.classList.contains("txtchunk")) {
+					//double nested
+					foundparent = selection.focusNode.parentElement.parentElement;
+				}
+				if (foundparent)
 				{
-					if(prev_hl_chunk!=null)
-					{
+					if (prev_hl_chunk != null) {
 						prev_hl_chunk.classList.remove("hlchunk");
 					}
-					prev_hl_chunk = selection.focusNode.parentElement;
+					prev_hl_chunk = foundparent;
 					prev_hl_chunk.classList.add("hlchunk");
 				}
+
 				idle_timer = 0;
 			}
 		}
@@ -14481,7 +14520,7 @@ Current version indicated by LITEVER below.
 			fulltxt = fulltxt.replace(/\[<\|d\|.+?\|d\|>\]/g, stripimg_replace_str);
 
 			//always filter comments - new format
-			fulltxt = fulltxt.replace(/\[<\|.+?\|>\]/g, ""); //remove normal comments too
+			fulltxt = fulltxt.replace(/\[<\|[\s\S]+?\|>\]/g, ""); //remove normal comments too
 
 		}
 		return fulltxt;
@@ -15402,7 +15441,7 @@ Current version indicated by LITEVER below.
 					inner = render_image_html(inner, "",false,true);
 					return inner;
 				});
-				processed_msg = processed_msg.replace(/\[<\|.+?\|>\]/g, ""); //remove normal comments too
+				processed_msg = processed_msg.replace(/\[<\|[\s\S]+?\|>\]/g, ""); //remove normal comments too
 			}
 
 			let namepart = (curr.myturn ? "User" : cosmetic_corpo_ai_nick);
@@ -15430,6 +15469,16 @@ Current version indicated by LITEVER below.
 					}
 				}
 
+				let foundTimestamp = "";
+				if(localsettings.inject_timestamps)
+				{
+					let found = processed_msg.match(/(\[\d{1,2}\/\d{1,2}\/\d{4}, \d{1,2}:\d{2} [AP]M\]) /g);
+					if(found && found.length>0)
+					{
+						foundTimestamp = found[0];
+						processed_msg = processed_msg.replace(/(\[\d{1,2}\/\d{1,2}\/\d{4}, \d{1,2}:\d{2} [AP]M\]) /g, "");
+					}
+				}
 				for(let i=0;i<validprefixes.length;++i)
 				{
 					let person = validprefixes[i];
@@ -15441,6 +15490,10 @@ Current version indicated by LITEVER below.
 						break;
 					}
 				}
+				if(foundTimestamp)
+				{
+					processed_msg = foundTimestamp + "\n" + processed_msg;
+				}
 			}
 
 			let bodypart = (corpo_editing_turn == i ?
@@ -15579,7 +15632,7 @@ Current version indicated by LITEVER below.
 		for(var i=0;i<input.length;++i)
 		{
 			let tempfullsearchable = input[i]; //strip out images
-			let txtwithnoimages = tempfullsearchable.replace(/\[<\|.+?\|>\]/g, "");
+			let txtwithnoimages = tempfullsearchable.replace(/\[<\|[\s\S]+?\|>\]/g, "");
 			var foundopponent = txtwithnoimages.match(othernamesregex);
 			var foundself = txtwithnoimages.match(mynameregex);
 
@@ -15668,7 +15721,7 @@ Current version indicated by LITEVER below.
 					inner = render_image_html(inner, "",false,true);
 					return inner;
 				});
-				curr.msg = curr.msg.replace(/\[<\|.+?\|>\]/g, ""); //remove normal comments too
+				curr.msg = curr.msg.replace(/\[<\|[\s\S]+?\|>\]/g, ""); //remove normal comments too
 
 
 			}
@@ -17709,6 +17762,11 @@ Current version indicated by LITEVER below.
 								class="helptext">Modifies the context, injecting tokens to improve adventure quality for new adventures.</span></span> </div>
 							<input type="checkbox" title="Adventure PrePrompt" id="adventure_context_mod" style="margin:0px 0px 0px auto;">
 						</div>
+						<div class="settinglabel">
+							<div class="justifyleft settingsmall">Fix Alpaca Leakage <span class="helpicon">?<span
+								class="helptext">Prevents leaking when Alpaca instruct format is used on models trained with bad/different formats.</span></span> </div>
+							<input type="checkbox" title="Fix Alpaca Leakage" id="fix_alpaca_leak" style="margin:0px 0px 0px auto;">
+						</div>
 					</div>
 
 					<div class="settingitem">
@@ -18192,6 +18250,12 @@ Current version indicated by LITEVER below.
 								</span>: </div>
 								<input title="Save Higher-Res Images" type="checkbox" id="img_allowhd" style="margin:0px 0 0;">
 							</div>
+							<div class="inlinelabel">
+								<div class="justifyleft" style="padding:4px">Crop Images <span class="helpicon">?
+									<span class="helptext">If enabled, oversized imported images will be cropped to fit. If disabled, images will be letterboxed instead.</span>
+								</span>: </div>
+								<input title="Crop Images" type="checkbox" id="img_crop" style="margin:0px 0 0;">
+							</div>
 						</div>
 					</div>
 				</div>
diff --git a/koboldcpp.py b/koboldcpp.py
index 22db95459..8b8a901f1 100644
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -768,16 +768,20 @@ def fetch_gpu_properties(testCL,testCU,testVK):
                         FetchedCUdeviceMem = [line.split(",")[1].strip() for line in getamdvram.splitlines()[1:] if line.strip()]
             except Exception as e:
                 pass
+        lowestcumem = 0
+        lowestfreecumem = 0
         for idx in range(0,4):
             if(len(FetchedCUdevices)>idx):
                 CUDevicesNames[idx] = FetchedCUdevices[idx]
                 if len(FetchedCUdeviceMem)>idx:
-                    if AMDgpu:
-                        MaxMemory[0] = max(int(FetchedCUdeviceMem[idx]),MaxMemory[0])
-                    else:
-                        MaxMemory[0] = max(int(FetchedCUdeviceMem[idx])*1024*1024,MaxMemory[0])
+                    dmem = int(FetchedCUdeviceMem[idx]) if AMDgpu else (int(FetchedCUdeviceMem[idx])*1024*1024)
+                    lowestcumem = dmem if lowestcumem==0 else (dmem if dmem<lowestcumem else lowestcumem)
                 if len(FetchedCUfreeMem)>idx:
-                    MaxFreeMemory[0] = max(int(FetchedCUfreeMem[idx])*1024*1024,MaxFreeMemory[0])
+                    dmem = (int(FetchedCUfreeMem[idx])*1024*1024)
+                    lowestfreecumem = dmem if lowestfreecumem==0 else (dmem if dmem<lowestfreecumem else lowestfreecumem)
+
+        MaxMemory[0] = max(lowestcumem,MaxMemory[0])
+        MaxFreeMemory[0] = max(lowestfreecumem,MaxFreeMemory[0])
 
     if testVK:
         try: # Get Vulkan names