diff --git a/src/inspector_agent.cc b/src/inspector_agent.cc
index 22506e26b32c..4c90afcf55fe 100644
--- a/src/inspector_agent.cc
+++ b/src/inspector_agent.cc
@@ -1051,15 +1051,9 @@ void Agent::ToggleNetworkTracking(Isolate* isolate, Local<Function> fn) {
 }
 
 void Agent::WaitForDisconnect() {
-  THROW_IF_INSUFFICIENT_PERMISSIONS(parent_env_,
-                                    permission::PermissionScope::kInspector,
-                                    "WaitForDisconnect");
-  if (!parent_env_->should_create_inspector() && !client_) {
-    ThrowUninitializedInspectorError(parent_env_);
+  if (client_ == nullptr) {
     return;
   }
-
-  CHECK_NOT_NULL(client_);
   bool is_worker = parent_handle_ != nullptr;
   parent_handle_.reset();
   if (client_->hasConnectedSessions() && !is_worker) {
diff --git a/src/permission/permission.cc b/src/permission/permission.cc
index 53d3a7657440..2071eefd8a4c 100644
--- a/src/permission/permission.cc
+++ b/src/permission/permission.cc
@@ -25,6 +25,7 @@ using v8::Context;
 using v8::DictionaryTemplate;
 using v8::FastApiCallbackOptions;
 using v8::FunctionCallbackInfo;
+using v8::HandleScope;
 using v8::IntegrityLevel;
 using v8::Local;
 using v8::MaybeLocal;
@@ -319,6 +320,10 @@ MaybeLocal<Value> CreateAccessDeniedError(Environment* env,
 void Permission::ThrowAccessDenied(Environment* env,
                                    PermissionScope perm,
                                    std::string_view res) {
+  if (!env->can_call_into_js()) {
+    return;
+  }
+  HandleScope handle_scope(env->isolate());
   Local<Value> err;
   if (CreateAccessDeniedError(env, perm, res).ToLocal(&err)) {
     env->isolate()->ThrowException(err);
@@ -331,6 +336,10 @@ void Permission::AsyncThrowAccessDenied(Environment* env,
                                         fs::FSReqBase* req_wrap,
                                         PermissionScope perm,
                                         std::string_view res) {
+  if (!env->can_call_into_js()) {
+    return;
+  }
+  HandleScope handle_scope(env->isolate());
   Local<Value> err;
   if (CreateAccessDeniedError(env, perm, res).ToLocal(&err)) {
     return req_wrap->Reject(err);
diff --git a/test/parallel/test-permission-inspector-drop-exit.js b/test/parallel/test-permission-inspector-drop-exit.js
new file mode 100644
index 000000000000..4886481976a3
--- /dev/null
+++ b/test/parallel/test-permission-inspector-drop-exit.js
@@ -0,0 +1,79 @@
+'use strict';
+const common = require('../common');
+
+common.skipIfInspectorDisabled();
+
+const assert = require('assert');
+const { spawn } = require('child_process');
+
+function runTest(script) {
+  return new Promise((resolve) => {
+    const child = spawn(
+      process.execPath,
+      [
+        '--permission',
+        '--allow-inspector',
+        '--allow-fs-read=*',
+        '--allow-fs-write=*',
+        '--inspect-wait=127.0.0.1:0',
+        '-e',
+        script,
+      ],
+      { stdio: ['ignore', 'pipe', 'pipe'] }
+    );
+
+    let stderr = '';
+    let socket;
+    let nextId = 0;
+
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+      const endpoint = stderr.match(
+        /Debugger listening on (ws:\/\/127\.0\.0\.1:\d+\/\S+)/
+      )?.[1];
+
+      if (endpoint && !socket) {
+        socket = new WebSocket(endpoint);
+        socket.onopen = () => {
+          for (const method of [
+            'Debugger.enable',
+            'Runtime.enable',
+            'Runtime.runIfWaitingForDebugger',
+          ]) {
+            socket.send(JSON.stringify({ id: ++nextId, method }));
+          }
+        };
+      }
+
+      if (stderr.includes('Waiting for the debugger to disconnect')) {
+        socket?.close();
+      }
+    });
+
+    child.on('close', common.mustCall((code, signal) => {
+      assert.strictEqual(
+        code,
+        0,
+        `Process exited with code ${code}, signal ${signal}, stderr: ${stderr}`
+      );
+      assert.strictEqual(signal, null, `Process killed with signal ${signal}`);
+      assert(
+        !stderr.includes('FATAL ERROR'),
+        `FATAL ERROR present in stderr: ${stderr}`
+      );
+      assert(
+        !stderr.includes('Cannot create a handle without a HandleScope'),
+        `HandleScope fatal present in stderr: ${stderr}`
+      );
+      resolve();
+    }));
+  });
+}
+
+async function main() {
+  await runTest("process.permission.drop('inspector');");
+  await runTest("process.permission.drop('inspector'); process.exit(0);");
+}
+
+main().then(common.mustCall());