diff --git a/src/core/document.js b/src/core/document.js
index 420fa00c3..3cba84d1a 100644
--- a/src/core/document.js
+++ b/src/core/document.js
@@ -33,7 +33,14 @@ import {
   warn,
 } from "../shared/util.js";
 import { Catalog, ObjectLoader, XRef } from "./obj.js";
-import { Dict, isDict, isName, isStream, Ref } from "./primitives.js";
+import {
+  clearPrimitiveCaches,
+  Dict,
+  isDict,
+  isName,
+  isStream,
+  Ref,
+} from "./primitives.js";
 import {
   getInheritableProperty,
   MissingDataException,
@@ -815,8 +822,8 @@ class PDFDocument {
     return this.catalog.fontFallback(id, handler);
   }
 
-  cleanup() {
-    return this.catalog.cleanup();
+  async cleanup() {
+    return this.catalog ? this.catalog.cleanup() : clearPrimitiveCaches();
   }
 }
 
diff --git a/src/core/evaluator.js b/src/core/evaluator.js
index b221c0171..2f8a88126 100644
--- a/src/core/evaluator.js
+++ b/src/core/evaluator.js
@@ -935,11 +935,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
 
         // Falling back to a default font to avoid completely broken rendering,
         // but note that there're no guarantees that things will look "correct".
-        fontRef = new Dict();
-        fontRef.set("BaseFont", Name.get("PDFJS-FallbackFont"));
-        fontRef.set("Type", Name.get("FallbackType"));
-        fontRef.set("Subtype", Name.get("FallbackType"));
-        fontRef.set("Encoding", Name.get("WinAnsiEncoding"));
+        fontRef = PartialEvaluator.getFallbackFontDict();
       }
 
       if (this.fontCache.has(fontRef)) {
@@ -3132,6 +3128,21 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
     }
   };
 
+  // TODO: Change this to a `static` getter, using shadowing, once
+  //       `PartialEvaluator` is converted to a proper class.
+  PartialEvaluator.getFallbackFontDict = function() {
+    if (this._fallbackFontDict) {
+      return this._fallbackFontDict;
+    }
+    const dict = new Dict();
+    dict.set("BaseFont", Name.get("PDFJS-FallbackFont"));
+    dict.set("Type", Name.get("FallbackType"));
+    dict.set("Subtype", Name.get("FallbackType"));
+    dict.set("Encoding", Name.get("WinAnsiEncoding"));
+
+    return (this._fallbackFontDict = dict);
+  };
+
   return PartialEvaluator;
 })();
 
diff --git a/src/core/worker.js b/src/core/worker.js
index 12e46fe73..74c5fc566 100644
--- a/src/core/worker.js
+++ b/src/core/worker.js
@@ -593,16 +593,22 @@ var WorkerMessageHandler = {
 
     handler.on("Terminate", function wphTerminate(data) {
       terminated = true;
+
+      const waitOn = [];
       if (pdfManager) {
         pdfManager.terminate(new AbortException("Worker was terminated."));
+
+        const cleanupPromise = pdfManager.cleanup();
+        waitOn.push(cleanupPromise);
+
         pdfManager = null;
+      } else {
+        clearPrimitiveCaches();
       }
       if (cancelXHRs) {
         cancelXHRs(new AbortException("Worker was terminated."));
       }
-      clearPrimitiveCaches();
 
-      var waitOn = [];
       WorkerTasks.forEach(function(task) {
         waitOn.push(task.finished);
         task.terminate();