diff --git a/src/Literate.jl b/src/Literate.jl
index 5003bc9f..ebd9ceaf 100644
--- a/src/Literate.jl
+++ b/src/Literate.jl
@@ -936,15 +936,24 @@ function execute_block(sb::Module, block::String; inputfile::String, fake_source
         end
     end
     popdisplay(disp) # IOCapture.capture has a try-catch so should always end up here
-    if c.error && !continue_on_error
-        error("""
-             $(sprint(showerror, c.value))
-             when executing the following code block from inputfile `$(Base.contractuser(inputfile))`
-
-             ```julia
-             $block
-             ```
-             """)
+    if c.error
+        if continue_on_error
+            err = c.value
+            if err isa LoadError # include_string may wrap error in LoadError
+                err = err.error
+            end
+            error_output = "ERROR: " * sprint(showerror, err)
+            return nothing, error_output, disp.data
+        else
+           error("""
+                 $(sprint(showerror, c.value))
+                 when executing the following code block from inputfile `$(Base.contractuser(inputfile))`
+
+                 ```julia
+                 $block
+                 ```
+                 """)
+        end
     end
     return c.value, c.output, disp.data
 end
diff --git a/test/runtests.jl b/test/runtests.jl
index af30d8e0..f7df492c 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1471,7 +1471,7 @@ end end
         write(inputfile, input_with_error)
         Literate.markdown(inputfile, sandbox; continue_on_error = true, execute = true)
         output_md = read(joinpath(sandbox, "input.md"), String)
-        @test occursin("DomainError(-1.0", output_md)
+        @test occursin("ERROR: DomainError with -1.0", output_md)
     end
 end