diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 024dbd80e2..099c34056d 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1180,6 +1180,8 @@ private:
             slot.smpl.reset();
         }
 
+        slot.task = std::make_unique<const server_task>(std::move(task));
+
         // initialize draft batch
         // TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
         if (slot.can_speculate()) {
@@ -1188,8 +1190,6 @@ private:
             slot.batch_spec = llama_batch_init(task.params.speculative.n_max + 1, 0, 1);
         }
 
-        slot.task = std::make_unique<const server_task>(std::move(task));
-
         slot.state = slot.task->is_child()
             ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
             : SLOT_STATE_STARTED;