diff --git a/book/cookbook/benchmarks/benchmark_tags_dont_execute.ipynb b/book/cookbook/benchmarks/benchmark_tags_dont_execute.ipynb
index 24e2d2c147f0b48a5af137121ea698a640dcf63b..011a55dfccb6d3a01ac0ff51229ceebd8240796c 100644
--- a/book/cookbook/benchmarks/benchmark_tags_dont_execute.ipynb
+++ b/book/cookbook/benchmarks/benchmark_tags_dont_execute.ipynb
@@ -43,7 +43,7 @@
    "execution_count": null,
    "metadata": {
     "tags": [
-     "thebe-remove-input"
+     "thebe-remove-input-init"
     ]
    },
    "outputs": [],
@@ -57,7 +57,7 @@
    "execution_count": 1,
    "metadata": {
     "tags": [
-     "thebe-remove-input"
+     "thebe-remove-input-init"
     ]
    },
    "outputs": [
diff --git a/book/cookbook/widgets_dont_execute.ipynb b/book/cookbook/widgets_dont_execute.ipynb
index f55ae6b41edf91317cfedcddfdbb312d378d6854..90b734122181a85f5d8616f0d0eb92c8617c960e 100644
--- a/book/cookbook/widgets_dont_execute.ipynb
+++ b/book/cookbook/widgets_dont_execute.ipynb
@@ -45,7 +45,7 @@
    "execution_count": null,
    "metadata": {
     "tags": [
-     "thebe-remove-input"
+     "thebe-remove-input-init"
     ]
    },
    "outputs": [],
@@ -169,7 +169,7 @@
    "execution_count": null,
    "metadata": {
     "tags": [
-     "thebe-remove-input"
+     "thebe-remove-input-init"
     ]
    },
    "outputs": [],
diff --git a/thebe_lite/_static/code.css b/thebe_lite/_static/code.css
index 5a2ec200259bad2ace2d5f28290d0f6a3978a678..ef17b2b3255e9f6c30034b7fe66dccb20b1b4de0 100644
--- a/thebe_lite/_static/code.css
+++ b/thebe_lite/_static/code.css
@@ -18,8 +18,8 @@
     color: var(--pst-color-text-base);
 }
 
-.jp-OutputArea {
-    padding: 5px 0px 5px 0px;
+.cell_output .jp-OutputArea-output pre {
+    border: var(--mystnb-source-border-width) var(--mystnb-source-border-color) solid;
 }
 
 .thebe-controls { 
@@ -53,6 +53,24 @@
   height: 20px;
 }
 
+html[data-theme=dark] .tag_hide-input summary {
+    background: var(--pst-color-surface) !important;
+}
+
+.cell_output {
+    margin-top: 0.2em !important;
+}
+
+.cell_output .output.stream {
+    margin-top: 0.2em !important;
+    border: var(--mystnb-source-border-width) var(--mystnb-source-border-color) solid !important;
+}
+
+div.container.cell_output {
+    padding-left: 0;
+    margin-bottom: 1em;
+}
+
 /* These rules are used for the dark mode code styling */
 /* Values taken from: https://codemirror.net/5/demo/theme.html#abcdef, with some minor tweaks to comments and gutter */
 html[data-theme=dark] div.CodeMirror-selected { background: #515151; }
diff --git a/thebe_lite/_static/sphinx-thebe.js b/thebe_lite/_static/sphinx-thebe.js
index 2c5d113b273905603c7add7c657bc803474fb6c7..4b9ff23b3a9511ebc4fbf669b84fddb7ebb3de9b 100644
--- a/thebe_lite/_static/sphinx-thebe.js
+++ b/thebe_lite/_static/sphinx-thebe.js
@@ -183,21 +183,6 @@ var configureThebe = () => {
   });
 };
 
-async function runInitCells() {
-  var thebeInitCells = document.querySelectorAll(
-    ".thebe-init, .tag_thebe-init"
-  );
-  for (const cell of thebeInitCells) {
-    console.log("Initializing Thebe with cell: " + cell.id);
-    const notebookCell = getNotebookCellOfCodeCell(cell);
-
-    // Emulate clicking the run button, with more information about execution
-    thebe.setButtonsBusy(notebookCell.id);
-    await notebookCell.execute();
-    thebe.clearButtonsBusy(notebookCell.id);
-  }
-}
-
 /**
  * Update the page DOM to use Thebe elements
  */
@@ -385,17 +370,46 @@ function override_pyodide_lookup(fs, server_path) {
   fs.lookupPath = new_lookup;
 }
 
+async function runInitCells() {
+  var thebeInitCells = document.querySelectorAll(
+    ".thebe-init, .tag_thebe-init"
+  );
+  for (const cell of thebeInitCells) {
+    console.log("Initializing Thebe with cell: " + cell.id);
+    const notebookCell = getNotebookCellOfCodeCell(cell);
+
+    // Emulate clicking the run button, with more information about execution
+    thebe.setButtonsBusy(notebookCell.id);
+    await notebookCell.execute();
+    thebe.clearButtonsBusy(notebookCell.id);
+  }
+}
+
+function wrapNakedOutput(element) {
+  const wrapper = document.createElement("div");
+  wrapper.classList.add("cell_output", "container");
+  wrapper.appendChild(element);
+  return wrapper;
+}
+
 function setupSpecialTaggedElements() {
   for (const taggedElement of window.specialTaggedElements) {
     switch (taggedElement.tag) {
-      case "thebe-remove-input": {
+      case "thebe-remove-input-init": {
         const { newCellInfo, newNotebookCell } = setupNewCell(
           undefined,
           undefined,
           taggedElement.code
         );
-        newNotebookCell.execute();
-        taggedElement.placeholder.before(newNotebookCell.area.node);
+
+        const wrappedOutput = wrapNakedOutput(newNotebookCell.area.node);
+        // The 4 following lines are an ugly hack to make sure we preserve init order
+        // Maybe improving runInitCells could circumvent this
+        wrappedOutput.classList.add("tag_thebe-init");
+        const idDiv = document.createElement("div");
+        idDiv.setAttribute("data-thebe-id", newNotebookCell.id);
+        wrappedOutput.appendChild(idDiv);
+        taggedElement.placeholder.before(wrappedOutput);
         break;
       }
       default: {
@@ -411,7 +425,7 @@ function moveHideInputOutput() {
   const taggedCells = document.querySelectorAll(".tag_hide-input");
   for (const cell of taggedCells) {
     const outputArea = cell.querySelector(".jp-OutputArea");
-    cell.after(outputArea);
+    cell.appendChild(wrapNakedOutput(outputArea));
   }
 }
 
@@ -420,15 +434,6 @@ var initThebe = async () => {
   document.querySelector(".dropdown-launch-buttons").remove();
 
   console.log("[sphinx-thebe]: Loading thebe...");
-  $(".thebe-launch-button ").css("display", "none");
-
-  // Provides nice things like a running animation and some padding
-  {
-    await loadStyleAsync("/thebe.css");
-    await loadStyleAsync("/_static/code.css");
-  }
-
-  $(".thebe-launch-button ").css("display", "block");
   $(".thebe-launch-button ").text("Loading thebe...");
 
   await loadScriptAsync("/thebe-lite.min.js");
@@ -461,14 +466,13 @@ var initThebe = async () => {
     code: `import ipykernel; ipykernel.version_info = (0,0)`,
   }).done;
   updateThebeButtonStatus("Running pre-intialized cells...");
+  setupSpecialTaggedElements();
 
   await runInitCells();
 
   updateThebeButtonStatus("Python interaction ready!", false);
 
   moveHideInputOutput();
-
-  setupSpecialTaggedElements();
 };
 
 // Helper function to munge the language name
@@ -486,7 +490,7 @@ function handleThebeRemoveInputTag(element) {
   placeholder.style.display = "none";
 
   window.specialTaggedElements.push({
-    tag: "thebe-remove-input",
+    tag: "thebe-remove-input-init",
     placeholder: placeholder,
     code: element.querySelector("pre").textContent?.trim() ?? "",
   });
@@ -500,10 +504,10 @@ function handleThebeRemoveInputTag(element) {
 }
 
 // Deal with custom-defined tags to properly prepare Thebe and DOM
-// Current special tags: thebe-remove-input
+// Current special tags: thebe-remove-input-init
 function consumeSpecialTags() {
   const specialTagsInfo = [
-    { tag: "thebe-remove-input", handler: handleThebeRemoveInputTag },
+    { tag: "thebe-remove-input-init", handler: handleThebeRemoveInputTag },
   ];
 
   window.specialTaggedElements = [];
@@ -524,3 +528,6 @@ if (document.readyState !== "loading") {
     consumeSpecialTags();
   });
 }
+
+loadStyleAsync("/thebe.css");
+loadStyleAsync("/_static/code.css");