mirrors/pdf.js
1
0
Fork 0
mirror of https://github.com/mozilla/pdf.js.git synced 2025-04-22 16:18:08 +02:00
Code Activity

Merge pull request #5263 from yurydelendik/stream

Browse source 
Implement streaming using moz-chunk-arraybuffer
This commit is contained in:
Brendan Dahl 2014-09-25 16:40:28 -07:00
commit 9c56c6f9f6
10 changed files with 282 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

76
src/core/chunked_stream.js
View file

@ -30,7 +30,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
this.numChunksLoaded = 0;
this.numChunks = Math.ceil(length / chunkSize);
this.manager = manager;
this.initialDataLength = 0;
this.progressiveDataLength = 0;
this.lastSuccessfulEnsureByteChunk = -1; // a single-entry cache
}
@ -41,7 +41,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
getMissingChunks: function ChunkedStream_getMissingChunks() {
var chunks = [];
for (var chunk = 0, n = this.numChunks; chunk < n; ++chunk) {
if (!(chunk in this.loadedChunks)) {
if (!this.loadedChunks[chunk]) {
chunks.push(chunk);
}
}
@ -73,21 +73,29 @@ var ChunkedStream = (function ChunkedStreamClosure() {
var curChunk;
for (curChunk = beginChunk; curChunk < endChunk; ++curChunk) {
if (!(curChunk in this.loadedChunks)) {
if (!this.loadedChunks[curChunk]) {
this.loadedChunks[curChunk] = true;
++this.numChunksLoaded;
}
}
},
onReceiveInitialData: function ChunkedStream_onReceiveInitialData(data) {
this.bytes.set(data);
this.initialDataLength = data.length;
var endChunk = (this.end === data.length ?
this.numChunks : Math.floor(data.length / this.chunkSize));
for (var i = 0; i < endChunk; i++) {
this.loadedChunks[i] = true;
++this.numChunksLoaded;
onReceiveProgressiveData:
function ChunkedStream_onReceiveProgressiveData(data) {
var position = this.progressiveDataLength;
var beginChunk = Math.floor(position / this.chunkSize);
this.bytes.set(new Uint8Array(data), position);
position += data.byteLength;
this.progressiveDataLength = position;
var endChunk = position >= this.end ? this.numChunks :
Math.floor(position / this.chunkSize);
var curChunk;
for (curChunk = beginChunk; curChunk < endChunk; ++curChunk) {
if (!this.loadedChunks[curChunk]) {
this.loadedChunks[curChunk] = true;
++this.numChunksLoaded;
}
}
},
@ -97,7 +105,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
return;
}
if (!(chunk in this.loadedChunks)) {
if (!this.loadedChunks[chunk]) {
throw new MissingDataException(pos, pos + 1);
}
this.lastSuccessfulEnsureByteChunk = chunk;
@ -108,7 +116,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
return;
}
if (end <= this.initialDataLength) {
if (end <= this.progressiveDataLength) {
return;
}
@ -116,7 +124,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
var beginChunk = Math.floor(begin / chunkSize);
var endChunk = Math.floor((end - 1) / chunkSize) + 1;
for (var chunk = beginChunk; chunk < endChunk; ++chunk) {
if (!(chunk in this.loadedChunks)) {
if (!this.loadedChunks[chunk]) {
throw new MissingDataException(begin, end);
}
}
@ -125,13 +133,13 @@ var ChunkedStream = (function ChunkedStreamClosure() {
nextEmptyChunk: function ChunkedStream_nextEmptyChunk(beginChunk) {
var chunk, n;
for (chunk = beginChunk, n = this.numChunks; chunk < n; ++chunk) {
if (!(chunk in this.loadedChunks)) {
if (!this.loadedChunks[chunk]) {
return chunk;
}
}
// Wrap around to beginning
for (chunk = 0; chunk < beginChunk; ++chunk) {
if (!(chunk in this.loadedChunks)) {
if (!this.loadedChunks[chunk]) {
return chunk;
}
}
@ -139,7 +147,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
},
hasChunk: function ChunkedStream_hasChunk(chunk) {
return chunk in this.loadedChunks;
return !!this.loadedChunks[chunk];
},
get length() {
@ -238,7 +246,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
var endChunk = Math.floor((this.end - 1) / chunkSize) + 1;
var missingChunks = [];
for (var chunk = beginChunk; chunk < endChunk; ++chunk) {
if (!(chunk in this.loadedChunks)) {
if (!this.loadedChunks[chunk]) {
missingChunks.push(chunk);
}
}
@ -300,28 +308,16 @@ var ChunkedStreamManager = (function ChunkedStreamManagerClosure() {
this.chunksNeededByRequest = {};
this.requestsByChunk = {};
this.callbacksByRequest = {};
this.progressiveDataLength = 0;
this._loadedStreamCapability = createPromiseCapability();
if (args.initialData) {
this.setInitialData(args.initialData);
this.onReceiveData({chunk: args.initialData});
}
}
ChunkedStreamManager.prototype = {
setInitialData: function ChunkedStreamManager_setInitialData(data) {
this.stream.onReceiveInitialData(data);
if (this.stream.allChunksLoaded()) {
this._loadedStreamCapability.resolve(this.stream);
} else if (this.msgHandler) {
this.msgHandler.send('DocProgress', {
loaded: data.length,
total: this.length
});
}
},
onLoadedStream: function ChunkedStreamManager_getLoadedStream() {
return this._loadedStreamCapability.promise;
},
@ -459,13 +455,21 @@ var ChunkedStreamManager = (function ChunkedStreamManagerClosure() {
onReceiveData: function ChunkedStreamManager_onReceiveData(args) {
var chunk = args.chunk;
var begin = args.begin;
var isProgressive = args.begin === undefined;
var begin = isProgressive ? this.progressiveDataLength : args.begin;
var end = begin + chunk.byteLength;
var beginChunk = this.getBeginChunk(begin);
var endChunk = this.getEndChunk(end);
var beginChunk = Math.floor(begin / this.chunkSize);
var endChunk = end < this.length ? Math.floor(end / this.chunkSize) :
Math.ceil(end / this.chunkSize);
if (isProgressive) {
this.stream.onReceiveProgressiveData(chunk);
this.progressiveDataLength = end;
} else {
this.stream.onReceiveData(begin, chunk);
}
this.stream.onReceiveData(begin, chunk);
if (this.stream.allChunksLoaded()) {
this._loadedStreamCapability.resolve(this.stream);
}

51
src/core/network.js
View file

@ -68,11 +68,11 @@ var NetworkManager = (function NetworkManagerClosure() {
return data;
}
var length = data.length;
var buffer = new Uint8Array(length);
var array = new Uint8Array(length);
for (var i = 0; i < length; i++) {
buffer[i] = data.charCodeAt(i) & 0xFF;
array[i] = data.charCodeAt(i) & 0xFF;
}
return buffer;
return array.buffer;
}
NetworkManager.prototype = {
@ -87,11 +87,11 @@ var NetworkManager = (function NetworkManagerClosure() {
return this.request(args);
},
requestFull: function NetworkManager_requestRange(listeners) {
requestFull: function NetworkManager_requestFull(listeners) {
return this.request(listeners);
},
request: function NetworkManager_requestRange(args) {
request: function NetworkManager_request(args) {
var xhr = this.getXhr();
var xhrId = this.currXhrId++;
var pendingRequest = this.pendingRequests[xhrId] = {
@ -115,27 +115,54 @@ var NetworkManager = (function NetworkManagerClosure() {
pendingRequest.expectedStatus = 200;
}
xhr.responseType = 'arraybuffer';
if (args.onProgress) {
xhr.onprogress = args.onProgress;
if (args.onProgressiveData) {
xhr.responseType = 'moz-chunked-arraybuffer';
if (xhr.responseType === 'moz-chunked-arraybuffer') {
pendingRequest.onProgressiveData = args.onProgressiveData;
pendingRequest.mozChunked = true;
} else {
xhr.responseType = 'arraybuffer';
}
} else {
xhr.responseType = 'arraybuffer';
}
if (args.onError) {
xhr.onerror = function(evt) {
args.onError(xhr.status);
};
}
xhr.onreadystatechange = this.onStateChange.bind(this, xhrId);
xhr.onprogress = this.onProgress.bind(this, xhrId);
pendingRequest.onHeadersReceived = args.onHeadersReceived;
pendingRequest.onDone = args.onDone;
pendingRequest.onError = args.onError;
pendingRequest.onProgress = args.onProgress;
xhr.send(null);
return xhrId;
},
onProgress: function NetworkManager_onProgress(xhrId, evt) {
var pendingRequest = this.pendingRequests[xhrId];
if (!pendingRequest) {
// Maybe abortRequest was called...
return;
}
if (pendingRequest.mozChunked) {
var chunk = getArrayBuffer(pendingRequest.xhr);
pendingRequest.onProgressiveData(chunk);
}
var onProgress = pendingRequest.onProgress;
if (onProgress) {
onProgress(evt);
}
},
onStateChange: function NetworkManager_onStateChange(xhrId, evt) {
var pendingRequest = this.pendingRequests[xhrId];
if (!pendingRequest) {
@ -196,6 +223,8 @@ var NetworkManager = (function NetworkManagerClosure() {
begin: begin,
chunk: chunk
});
} else if (pendingRequest.onProgressiveData) {
pendingRequest.onDone(null);
} else {
pendingRequest.onDone({
begin: 0,
@ -215,6 +244,10 @@ var NetworkManager = (function NetworkManagerClosure() {
return this.pendingRequests[xhrId].xhr;
},
isStreamingRequest: function NetworkManager_isStreamingRequest(xhrId) {
return !!(this.pendingRequests[xhrId].onProgressiveData);
},
isPendingRequest: function NetworkManager_isPendingRequest(xhrId) {
return xhrId in this.pendingRequests;
},

9
src/core/pdf_manager.js
View file

@ -65,6 +65,10 @@ var BasePdfManager = (function BasePdfManagerClosure() {
return new NotImplementedException();
},
sendProgressiveData: function BasePdfManager_sendProgressiveData(chunk) {
return new NotImplementedException();
},
updatePassword: function BasePdfManager_updatePassword(password) {
this.pdfDocument.xref.password = this.password = password;
if (this._passwordChangedCapability) {
@ -201,6 +205,11 @@ var NetworkPdfManager = (function NetworkPdfManagerClosure() {
this.streamManager.requestAllChunks();
};
NetworkPdfManager.prototype.sendProgressiveData =
function NetworkPdfManager_sendProgressiveData(chunk) {
this.streamManager.onReceiveData({ chunk: chunk });
};
NetworkPdfManager.prototype.onLoadedStream =
function NetworkPdfManager_getLoadedStream() {
return this.streamManager.onLoadedStream();

57
src/core/worker.js
View file

@ -16,7 +16,7 @@
*/
/* globals PDFJS, createPromiseCapability, LocalPdfManager, NetworkPdfManager,
NetworkManager, isInt, RANGE_CHUNK_SIZE, MissingPDFException,
UnexpectedResponseException, PasswordException, Promise,
UnexpectedResponseException, PasswordException, Promise, warn,
PasswordResponses, InvalidPDFException, UnknownErrorException,
XRefParseException, Ref, info, globalScope, error, MessageHandler */
@ -86,6 +86,7 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
httpHeaders: source.httpHeaders,
withCredentials: source.withCredentials
});
var cachedChunks = [];
var fullRequestXhrId = networkManager.requestFull({
onHeadersReceived: function onHeadersReceived() {
if (disableRange) {
@ -116,11 +117,18 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
return;
}
// NOTE: by cancelling the full request, and then issuing range
// requests, there will be an issue for sites where you can only
// request the pdf once. However, if this is the case, then the
// server should not be returning that it can support range requests.
networkManager.abortRequest(fullRequestXhrId);
if (networkManager.isStreamingRequest(fullRequestXhrId)) {
// We can continue fetching when progressive loading is enabled,
// and we don't need the autoFetch feature.
source.disableAutoFetch = true;
} else {
// NOTE: by cancelling the full request, and then issuing range
// requests, there will be an issue for sites where you can only
// request the pdf once. However, if this is the case, then the
// server should not be returning that it can support range
// requests.
networkManager.abortRequest(fullRequestXhrId);
}
try {
pdfManager = new NetworkPdfManager(source, handler);
@ -130,10 +138,44 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
}
},
onProgressiveData: PDFJS.disableStream ? null :
function onProgressiveData(chunk) {
if (!pdfManager) {
cachedChunks.push(chunk);
return;
}
pdfManager.sendProgressiveData(chunk);
},
onDone: function onDone(args) {
if (pdfManager) {
return; // already processed
}
var pdfFile;
if (args === null) {
// TODO add some streaming manager, e.g. for unknown length files.
// The data was returned in the onProgressiveData, combining...
var pdfFileLength = 0, pos = 0;
cachedChunks.forEach(function (chunk) {
pdfFileLength += chunk.byteLength;
});
if (source.length && pdfFileLength !== source.length) {
warn('reported HTTP length is different from actual');
}
var pdfFileArray = new Uint8Array(pdfFileLength);
cachedChunks.forEach(function (chunk) {
pdfFileArray.set(new Uint8Array(chunk), pos);
pos += chunk.byteLength;
});
pdfFile = pdfFileArray.buffer;
} else {
pdfFile = args.chunk;
}
// the data is array, instantiating directly from it
try {
pdfManager = new LocalPdfManager(args.chunk, source.password);
pdfManager = new LocalPdfManager(pdfFile, source.password);
pdfManagerCapability.resolve();
} catch (ex) {
pdfManagerCapability.reject(ex);
@ -228,6 +270,7 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
PDFJS.cMapPacked = data.cMapPacked === true;
getPdfManager(data).then(function () {
handler.send('PDFManagerReady', null);
pdfManager.onLoadedStream().then(function(stream) {
handler.send('DataLoaded', { length: stream.bytes.byteLength });
});

20
src/display/api.js
View file

@ -86,6 +86,14 @@ PDFJS.workerSrc = (PDFJS.workerSrc === undefined ? null : PDFJS.workerSrc);
PDFJS.disableRange = (PDFJS.disableRange === undefined ?
false : PDFJS.disableRange);
/**
* Disable streaming of PDF file data. By default PDF.js attempts to load PDF
* in chunks. This default behavior can be disabled.
* @var {boolean}
*/
PDFJS.disableStream = (PDFJS.disableStream === undefined ?
false : PDFJS.disableStream);
/**
* Disable pre-fetching of PDF file data. When range requests are enabled PDF.js
* will automatically keep fetching more data even if it isn't needed to display
@ -851,6 +859,12 @@ var WorkerTransport = (function WorkerTransportClosure() {
});
});
pdfDataRangeTransport.addProgressiveReadListener(function(chunk) {
messageHandler.send('OnDataRange', {
chunk: chunk
});
});
messageHandler.on('RequestDataRange',
function transportDataRange(data) {
pdfDataRangeTransport.requestDataRange(data.begin, data.end);
@ -911,6 +925,12 @@ var WorkerTransport = (function WorkerTransportClosure() {
this.downloadInfoCapability.resolve(data);
}, this);
messageHandler.on('PDFManagerReady', function transportPage(data) {
if (this.pdfDataRangeTransport) {
this.pdfDataRangeTransport.transportReady();
}
}, this);
messageHandler.on('StartRenderPage', function transportRender(data) {
var page = this.pageCache[data.pageIndex];