From f510301922539ae9566da2d48ff4b1a7acea1fd3 Mon Sep 17 00:00:00 2001
From: Tom <tmpearce@gmail.com>
Date: Wed, 19 Jul 2023 18:18:56 -0400
Subject: [PATCH] add performance test demo page. reduce number of drawing
calls drawing pipeline of webgl drawer.
---
src/canvasdrawer.js | 4 +-
src/webgldrawer.js | 499 ++++++++++++++++++++---------
test/demo/drawercomparison.html | 1 +
test/demo/drawerperformance.html | 47 +++
test/demo/drawerperformance.js | 90 ++++++
test/demo/webgldemodrawer.js | 525 +++++++++++++++++++++----------
6 files changed, 842 insertions(+), 324 deletions(-)
create mode 100644 test/demo/drawerperformance.html
create mode 100644 test/demo/drawerperformance.js
diff --git a/src/canvasdrawer.js b/src/canvasdrawer.js
index be0e4cbb..6366e431 100644
--- a/src/canvasdrawer.js
+++ b/src/canvasdrawer.js
@@ -324,7 +324,9 @@ class CanvasDrawer extends $.DrawerBase{
if (tiledImage._croppingPolygons) {
var self = this;
- this._saveContext(useSketch);
+ if(!usedClip){
+ this._saveContext(useSketch);
+ }
try {
var polygons = tiledImage._croppingPolygons.map(function (polygon) {
return polygon.map(function (coord) {
diff --git a/src/webgldrawer.js b/src/webgldrawer.js
index d634b260..4b35e19f 100644
--- a/src/webgldrawer.js
+++ b/src/webgldrawer.js
@@ -1,3 +1,4 @@
+
/*
* OpenSeadragon - WebGLDrawer
*
@@ -142,11 +143,10 @@
this._TileMap = new Map();
this._gl = null;
- this._glLocs = null;
- this._glProgram = null;
- this._glUnitQuadBuffer = null;
+ this._firstPass = null;
+ this._secondPass = null;
this._glFrameBuffer = null;
- this._glTiledImageTexture = null;
+ this._renderToTexture = null;
this._glFramebufferToCanvasTransform = null;
this._outputCanvas = null;
this._outputContext = null;
@@ -196,7 +196,7 @@
});
// Delete all our created resources
- gl.deleteBuffer(this._glUnitQuadBuffer);
+ gl.deleteBuffer(this._secondPass.bufferOutputPosition);
gl.deleteFramebuffer(this._glFrameBuffer);
// TO DO: if/when render buffers or frame buffers are used, release them:
// gl.deleteRenderbuffer(someRenderbuffer);
@@ -282,35 +282,59 @@
let rotMatrix = Mat3.makeRotation(-viewport.rotation);
let viewMatrix = scaleMatrix.multiply(rotMatrix).multiply(posMatrix);
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+
// clear the output canvas
this._outputContext.clearRect(0, 0, this._outputCanvas.width, this._outputCanvas.height);
- // TO DO: further optimization is possible.
- // If no clipping and no composite operation, the tiled images
- // can all be drawn onto the rendering canvas at the same time, avoiding
- // unnecessary clearing and copying of the pixel data.
- // For now, I'm doing it this way to replicate full functionality
- // of the context2d drawer
+ let renderingBufferHasImageData = false;
+
+ //iterate over tiled images and draw each one using a two-pass rendering pipeline if needed
+ tiledImages.forEach( (tiledImage, tiledImageIndex) => {
+
+ let useContext2dPipeline = ( tiledImage.compositeOperation ||
+ this.viewer.compositeOperation ||
+ tiledImage._clip ||
+ tiledImage._croppingPolygons ||
+ tiledImage.debugMode
+ );
+ let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1); // TO DO: check hasTransparency in addition to opacity
- //iterate over tiled imagesget the list of tiles to draw
- tiledImages.forEach( (tiledImage, i) => {
- //get the list of tiles to draw
let tilesToDraw = tiledImage.getTilesToDraw();
if(tilesToDraw.length === 0){
return;
}
- // bind to the framebuffer for render-to-texture
- gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
+ // using the context2d pipeline requires a clean rendering (back) buffer to start
+ if(useContext2dPipeline){
+ // if the rendering buffer has image data currently, write it to the output canvas now and clear it
- // clear the buffer
- gl.clear(gl.COLOR_BUFFER_BIT);
+ if(renderingBufferHasImageData){
+ this._outputContext.drawImage(this._renderingCanvas, 0, 0);
+ }
+
+ // clear the buffer
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+ }
+
+ // First rendering pass: compose tiles that make up this tiledImage
+ gl.useProgram(this._firstPass.shaderProgram);
+
+ // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null)
+ if(useTwoPassRendering){
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
+ // clear the buffer to draw a new image
+ gl.clear(gl.COLOR_BUFFER_BIT);
+ } else {
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ // no need to clear, just draw on top of the existing pixels
+ }
- // set opacity for this image
- gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity);
@@ -329,58 +353,142 @@
overallMatrix = viewMatrix.multiply(localMatrix);
}
- // iterate over tiles and draw each one to the buffer
- for(let ti = 0; ti < tilesToDraw.length; ti++){
- let tile = tilesToDraw[ti].tile;
- let textureInfo = this._TextureMap.get(tile.getCanvasContext().canvas);
+ let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS);
+ let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex
+ let textureDataArray = new Array(maxTextures);
+ let matrixArray = new Array(maxTextures);
+ let opacityArray = new Array(maxTextures);
+
+ // iterate over tiles and add data for each one to the buffers
+ for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){
+ let tile = tilesToDraw[tileIndex].tile;
+ let index = tileIndex % maxTextures;
+ let tileContext = tile.getCanvasContext();
+
+ let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null;
if(textureInfo){
- this._drawTile(tile, tiledImage, textureInfo, overallMatrix, tiledImage.opacity);
+ this._getTileData(tile, tiledImage, textureInfo, overallMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray);
} else {
// console.log('No tile info', tile);
}
- }
+ if( (index === maxTextures - 1) || (tileIndex === tilesToDraw.length - 1)){
+ // We've filled up the buffers: time to draw this set of tiles
+ // bind each tile's texture to the appropriate gl.TEXTURE#
+ for(let i = 0; i <= index; i++){
+ gl.activeTexture(gl.TEXTURE0 + i);
+ gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]);
+ }
- // Draw from the Framebuffer onto the rendering canvas buffer
+ // set the buffer data for the texture coordinates to use for each tile
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
+ gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW);
- gl.flush(); // finish drawing to the texture
- gl.bindFramebuffer(gl.FRAMEBUFFER, null); // null means bind to the backbuffer for drawing
- gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); // bind the rendered texture to use
- gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+ // set the transform matrix uniform for each tile
+ matrixArray.forEach( (matrix, index) => {
+ gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix);
+ });
+ // set the opacity uniform for each tile
+ gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray));
- // set up the matrix to draw the whole framebuffer to the entire clip space
- gl.uniformMatrix3fv(this._glLocs.uMatrix, false, this._glFramebufferToCanvasTransform);
+ // bind vertex buffers and (re)set attributes before calling gl.drawArrays()
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition);
+ gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
- // reset texturebuffer to unit quad
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW);
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
+ gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
- // set opacity to the value for the current tiledImage
- this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity);
- gl.drawArrays(gl.TRIANGLES, 0, 6);
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex);
+ gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0);
- // iterate over any filters - filters can use this._glTiledImageTexture to get rendered data if desired
- let filters = this.filters || [];
- for(let fi = 0; fi < filters.length; fi++){
- let filter = this.filters[fi];
- if(filter.apply){
- filter.apply(gl); // filter.apply should write data on top of the backbuffer (bound above)
+ // Draw! 6 vertices per tile (2 triangles per rectangle)
+ gl.drawArrays(gl.TRIANGLES, 0, 6 * (index + 1) );
}
}
- gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary?
- // draw from the rendering canvas onto the output canvas, clipping/cropping if needed.
- this._renderToOutputCanvas(tiledImage, tilesToDraw, i);
+ // gl.flush(); // is this necessary?
+
+ if(useTwoPassRendering){
+ // Second rendering pass: Render the tiled image from the framebuffer into the back buffer
+ gl.useProgram(this._secondPass.shaderProgram);
+
+ // set the rendering target to the back buffer (null)
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+
+ // bind the rendered texture from the first pass to use during this second pass
+ gl.activeTexture(gl.TEXTURE0);
+ gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
+
+ // set opacity to the value for the current tiledImage
+ this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity);
+
+ // bind buffers and set attributes before calling gl.drawArrays
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition);
+ gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition);
+ gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
+
+ // Draw the quad (two triangles)
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
+
+ // TO DO: is this the mechanism we want to use here?
+ // iterate over any filters - filters can use this._renderToTexture to get rendered data if desired
+ let filters = this.filters || [];
+ for(let fi = 0; fi < filters.length; fi++){
+ let filter = this.filters[fi];
+ if(filter.apply){
+ filter.apply(gl); // filter.apply should write data on top of the backbuffer (bound above)
+ }
+ }
+ }
+
+ renderingBufferHasImageData = true;
+
+ // gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary?
+
+ if(useContext2dPipeline){
+ // draw from the rendering canvas onto the output canvas, clipping/cropping if needed.
+ this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex);
+ renderingBufferHasImageData = false;
+ // clear the buffer
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+ }
+
+ // Fire tiled-image-drawn event.
+ // TO DO: the image data may not be on the output canvas yet!!
+ if( this.viewer ){
+ /**
+ * Raised when a tiled image is drawn to the canvas. Only valid
+ * for webgl drawer.
+ *
+ * @event tiled-image-drawn
+ * @memberof OpenSeadragon.Viewer
+ * @type {object}
+ * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event.
+ * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn.
+ * @property {Array} tiles - An array of Tile objects that were drawn.
+ * @property {?Object} userData - Arbitrary subscriber-defined object.
+ */
+ this.viewer.raiseEvent( 'tiled-image-drawn', {
+ tiledImage: tiledImage,
+ tiles: tilesToDraw.map(info => info.tile),
+ });
+ }
});
+ // TO DO: the line below is a test!
+ if(renderingBufferHasImageData){
+ this._outputContext.drawImage(this._renderingCanvas, 0, 0);
+ }
}
// Public API required by all Drawer implementations
/**
- * Set the context2d imageSmoothingEnabled parameter
- * @param {Boolean} enabled
- */
+ * Set the context2d imageSmoothingEnabled parameter
+ * @param {Boolean} enabled
+ */
setImageSmoothingEnabled(enabled){
this._clippingContext.imageSmoothingEnabled = enabled;
this._outputContext.imageSmoothingEnabled = enabled;
@@ -412,13 +520,13 @@
}
/**
- * Draw data from the rendering canvas onto the output canvas, with clipping,
- * cropping and/or debug info as requested.
- * @private
- * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw
- * @param {Array} tilesToDraw - array of objects containing tiles that were drawn
- */
- _renderToOutputCanvas(tiledImage, tilesToDraw, tiledImageIndex){
+ * Draw data from the rendering canvas onto the output canvas, with clipping,
+ * cropping and/or debug info as requested.
+ * @private
+ * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw
+ * @param {Array} tilesToDraw - array of objects containing tiles that were drawn
+ */
+ _applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex){
// composite onto the output canvas, clipping if necessary
this._outputContext.save();
@@ -439,39 +547,19 @@
this._drawDebugInfo(tilesToDraw, tiledImage, strokeStyle, fillStyle);
}
- // Fire tiled-image-drawn event now that the data is on the output canvas
- if( this.viewer ){
- /**
- * Raised when a tiled image is drawn to the canvas. Only valid
- * for webgl drawer.
- *
- * @event tiled-image-drawn
- * @memberof OpenSeadragon.Viewer
- * @type {object}
- * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event.
- * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn.
- * @property {Array} tiles - An array of Tile objects that were drawn.
- * @property {?Object} userData - Arbitrary subscriber-defined object.
- */
- this.viewer.raiseEvent( 'tiled-image-drawn', {
- tiledImage: tiledImage,
- tiles: tilesToDraw.map(info => info.tile),
- });
- }
+
}
// private
- _drawTile(tile, tiledImage, textureInfo, viewMatrix, imageOpacity){
+ _getTileData(tile, tiledImage, textureInfo, viewMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray){
- let gl = this._gl;
let texture = textureInfo.texture;
let textureQuad = textureInfo.position;
- // set the vertices into the non-overlapped portion of the texture
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, textureQuad, gl.DYNAMIC_DRAW);
+ // set the position of this texture
+ texturePositionArray.set(textureQuad, index * 12);
- // compute offsets for overlap
+ // compute offsets that account for tile overlap; needed for calculating the transform matrix appropriately
let overlapFraction = this._calculateOverlapFraction(tile, tiledImage);
let xOffset = tile.positionedBounds.width * overlapFraction.x;
let yOffset = tile.positionedBounds.height * overlapFraction.y;
@@ -502,41 +590,164 @@
let overallMatrix = viewMatrix.multiply(matrix);
- // set opacity for this image
- this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tile.opacity); // imageOpacity *
-
- gl.uniformMatrix3fv(this._glLocs.uMatrix, false, overallMatrix.values);
- gl.bindTexture(gl.TEXTURE_2D, texture);
+ opacityArray[index] = tile.opacity;// * tiledImage.opacity;
+ textureDataArray[index] = texture;
+ matrixArray[index] = overallMatrix.values;
if(this.continuousTileRefresh){
- // Upload the image into the texture (already bound to TEXTURE_2D above)
+ // Upload the image into the texture
+ // TO DO: test if this works appropriately
let tileContext = tile.getCanvasContext();
this._raiseTileDrawingEvent(tiledImage, this._outputContext, tile, tileContext);
this._uploadImageData(tileContext, tile, tiledImage);
}
-
- gl.drawArrays(gl.TRIANGLES, 0, 6);
}
_setupRenderer(){
-
- if(!this._gl){
+ let gl = this._gl;
+ if(!gl){
$.console.error('_setupCanvases must be called before _setupRenderer');
}
+ this._unitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1); // used a few places; create once and store the result
+ this._makeFirstPassShaderProgram();
+ this._makeSecondPassShaderProgram();
+
+ // set up the texture to render to in the first pass, and which will be used for rendering the second pass
+ this._renderToTexture = gl.createTexture();
+ gl.activeTexture(gl.TEXTURE0);
+ gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+ // set up the framebuffer for render-to-texture
+ this._glFrameBuffer = gl.createFramebuffer();
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
+ gl.framebufferTexture2D(
+ gl.FRAMEBUFFER,
+ gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0
+ gl.TEXTURE_2D, // attach a 2D texture
+ this._renderToTexture, // the texture to attach
+ 0
+ );
+
+ gl.enable(gl.BLEND);
+ gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
+
+ }
+
+ _makeFirstPassShaderProgram(){
+ let numTextures = this._glNumTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS);
+ let makeMatrixUniforms = () => {
+ return [...Array(numTextures).keys()].map(index => `uniform mat3 u_matrix_${index};`).join('\n');
+ };
+ let makeConditionals = () => {
+ return [...Array(numTextures).keys()].map(index => `${index > 0 ? 'else ' : ''}if(int(a_index) == ${index}) { transform_matrix = u_matrix_${index}; }`).join('\n');
+ };
+
+ const vertexShaderProgram = `
+ attribute vec2 a_output_position;
+ attribute vec2 a_texture_position;
+ attribute float a_index;
+
+ ${makeMatrixUniforms()} // create a uniform mat3 for each potential tile to draw
+
+ varying vec2 v_texture_position;
+ varying float v_image_index;
+
+ void main() {
+
+ mat3 transform_matrix; // value will be set by the if/elses in makeConditional()
+
+ ${makeConditionals()}
+
+ gl_Position = vec4(transform_matrix * vec3(a_output_position, 1), 1);
+
+ v_texture_position = a_texture_position;
+ v_image_index = a_index;
+ }
+ `;
+
+ const fragmentShaderProgram = `
+ precision mediump float;
+
+ // our textures
+ uniform sampler2D u_images[${numTextures}];
+ // our opacities
+ uniform float u_opacities[${numTextures}];
+
+ // the varyings passed in from the vertex shader.
+ varying vec2 v_texture_position;
+ varying float v_image_index;
+
+ void main() {
+ // can't index directly with a variable, need to use a loop iterator hack
+ for(int i = 0; i < ${numTextures}; ++i){
+ if(i == int(v_image_index)){
+ gl_FragColor = texture2D(u_images[i], v_texture_position) * u_opacities[i];
+ }
+ }
+ }
+ `;
+
+ let gl = this._gl;
+
+ let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram);
+ gl.useProgram(program);
+
+ // get locations of attributes and uniforms, and create buffers for each attribute
+ this._firstPass = {
+ shaderProgram: program,
+ aOutputPosition: gl.getAttribLocation(program, 'a_output_position'),
+ aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'),
+ aIndex: gl.getAttribLocation(program, 'a_index'),
+ uTransformMatrices: [...Array(this._glNumTextures).keys()].map(i=>gl.getUniformLocation(program, `u_matrix_${i}`)),
+ uImages: gl.getUniformLocation(program, 'u_images'),
+ uOpacities: gl.getUniformLocation(program, 'u_opacities'),
+ bufferOutputPosition: gl.createBuffer(),
+ bufferTexturePosition: gl.createBuffer(),
+ bufferIndex: gl.createBuffer(),
+ };
+
+ gl.uniform1iv(this._firstPass.uImages, [...Array(numTextures).keys()]);
+
+ // provide coordinates for the rectangle in output space, i.e. a unit quad for each one.
+ let outputQuads = new Float32Array(numTextures * 12);
+ for(let i = 0; i < numTextures; ++i){
+ outputQuads.set(Float32Array.from(this._unitQuad), i * 12);
+ }
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition);
+ gl.bufferData(gl.ARRAY_BUFFER, outputQuads, gl.STATIC_DRAW); // bind data statically here, since it's unchanging
+ gl.enableVertexAttribArray(this._firstPass.aOutputPosition);
+
+ // provide texture coordinates for the rectangle in image (texture) space. Data will be set later.
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
+ gl.enableVertexAttribArray(this._firstPass.aTexturePosition);
+
+ // for each vertex, provide an index into the array of textures/matrices to use for the correct tile
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex);
+ let indices = [...Array(this._glNumTextures).keys()].map(i => Array(6).fill(i)).flat(); // repeat each index 6 times, for the 6 vertices per tile (2 triangles)
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(indices), gl.STATIC_DRAW); // bind data statically here, since it's unchanging
+ gl.enableVertexAttribArray(this._firstPass.aIndex);
+
+ }
+
+ _makeSecondPassShaderProgram(){
const vertexShaderProgram = `
attribute vec2 a_output_position;
attribute vec2 a_texture_position;
uniform mat3 u_matrix;
- varying vec2 v_texCoord;
+ varying vec2 v_texture_position;
void main() {
- gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1);
+ gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1);
- v_texCoord = a_texture_position;
+ v_texture_position = a_texture_position;
}
`;
@@ -547,65 +758,48 @@
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
- varying vec2 v_texCoord;
+ varying vec2 v_texture_position;
// the opacity multiplier for the image
uniform float u_opacity_multiplier;
void main() {
- gl_FragColor = texture2D(u_image, v_texCoord);
- gl_FragColor *= u_opacity_multiplier;
+ gl_FragColor = texture2D(u_image, v_texture_position);
+ gl_FragColor *= u_opacity_multiplier;
}
`;
- let gl = this._gl;
- this._glProgram = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram);
- gl.useProgram(this._glProgram);
- gl.enable(gl.BLEND);
- gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
- this._glLocs = {
- aOutputPosition: gl.getAttribLocation(this._glProgram, 'a_output_position'),
- aTexturePosition: gl.getAttribLocation(this._glProgram, 'a_texture_position'),
- uMatrix: gl.getUniformLocation(this._glProgram, 'u_matrix'),
- uImage: gl.getUniformLocation(this._glProgram, 'u_image'),
- uOpacityMultiplier: gl.getUniformLocation(this._glProgram, 'u_opacity_multiplier')
+ let gl = this._gl;
+
+ let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram);
+ gl.useProgram(program);
+
+ // get locations of attributes and uniforms, and create buffers for each attribute
+ this._secondPass = {
+ shaderProgram: program,
+ aOutputPosition: gl.getAttribLocation(program, 'a_output_position'),
+ aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'),
+ uMatrix: gl.getUniformLocation(program, 'u_matrix'),
+ uImage: gl.getUniformLocation(program, 'u_image'),
+ uOpacityMultiplier: gl.getUniformLocation(program, 'u_opacity_multiplier'),
+ bufferOutputPosition: gl.createBuffer(),
+ bufferTexturePosition: gl.createBuffer(),
};
- this._glUnitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1);
- // provide texture coordinates for the rectangle in output space.
- this._glUnitQuadBuffer = gl.createBuffer(); //keep reference to clear it later
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glUnitQuadBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.STATIC_DRAW);
- gl.enableVertexAttribArray(this._glLocs.aOutputPosition);
- gl.vertexAttribPointer(this._glLocs.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
+
+ // provide coordinates for the rectangle in output space, i.e. a unit quad for each one.
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition);
+ gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.STATIC_DRAW); // bind data statically here since it's unchanging
+ gl.enableVertexAttribArray(this._secondPass.aOutputPosition);
// provide texture coordinates for the rectangle in image (texture) space.
- this._glTextureBuffer = gl.createBuffer(); //keep reference to clear it later
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW); // use unit quad to start, will be updated per tile
- gl.enableVertexAttribArray(this._glLocs.aTexturePosition);
- gl.vertexAttribPointer(this._glLocs.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
-
- // setup the framebuffer
- this._glTiledImageTexture = gl.createTexture();
- gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture);
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
- this._glFrameBuffer = gl.createFramebuffer();
- gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
- gl.framebufferTexture2D(
- gl.FRAMEBUFFER,
- gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0
- gl.TEXTURE_2D, // attach a 2D texture
- this._glTiledImageTexture, // the texture to attach
- 0
- );
-
- this._glFramebufferToCanvasTransform = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5)).values;
-
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition);
+ gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.DYNAMIC_DRAW); // bind data statically here since it's unchanging
+ gl.enableVertexAttribArray(this._secondPass.aTexturePosition);
+ // set the matrix that transforms the framebuffer to clip space
+ let matrix = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5));
+ gl.uniformMatrix3fv(this._secondPass.uMatrix, false, matrix.values);
}
_resizeRenderer(){
@@ -615,10 +809,11 @@
gl.viewport(0, 0, w, h);
//release the old texture
- gl.deleteTexture(this._glTiledImageTexture);
+ gl.deleteTexture(this._renderToTexture);
//create a new texture and set it up
- this._glTiledImageTexture = gl.createTexture();
- gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture);
+ this._renderToTexture = gl.createTexture();
+ gl.activeTexture(gl.TEXTURE0);
+ gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, w, h, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
@@ -626,7 +821,7 @@
//bind the frame buffer to the new texture
gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._glTiledImageTexture, 0);
+ gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderToTexture, 0);
}
@@ -708,7 +903,7 @@
position = this._makeQuadVertexBuffer(left, right, top, bottom);
} else {
// no overlap: this texture can use the unit quad as it's position data
- position = this._glUnitQuad;
+ position = this._unitQuad;
}
let textureInfo = {
@@ -718,7 +913,7 @@
// add it to our _TextureMap
this._TextureMap.set(canvas, textureInfo);
-
+ gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
// Set the parameters so we can render any size image.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
@@ -1035,14 +1230,6 @@
};
- // $.WebGLDrawer.Filter = class{
- // constructor(gl){
- // this.gl = gl;
- // }
- // apply(){
-
- // }
- // };
}( OpenSeadragon ));
diff --git a/test/demo/drawercomparison.html b/test/demo/drawercomparison.html
index 7c09a3c0..3e007419 100644
--- a/test/demo/drawercomparison.html
+++ b/test/demo/drawercomparison.html
@@ -6,6 +6,7 @@
<script type="text/javascript" src='../lib/jquery-1.9.1.min.js'></script>
<script type="text/javascript" src='../lib/jquery-ui-1.10.2/js/jquery-ui-1.10.2.min.js'></script>
<link rel="stylesheet" href="../lib/jquery-ui-1.10.2/css/smoothness/jquery-ui-1.10.2.min.css">
+ <script type="text/javascript" src="./webgldemodrawer.js"></script>
<script type="module" src="./drawercomparison.js"></script>
<style type="text/css">
diff --git a/test/demo/drawerperformance.html b/test/demo/drawerperformance.html
new file mode 100644
index 00000000..34de7659
--- /dev/null
+++ b/test/demo/drawerperformance.html
@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Drawer Comparison Demo</title>
+ <script type="text/javascript" src='../../build/openseadragon/openseadragon.js'></script>
+ <script type="text/javascript" src='../lib/jquery-1.9.1.min.js'></script>
+ <script type="text/javascript" src='../lib/jquery-ui-1.10.2/js/jquery-ui-1.10.2.min.js'></script>
+ <link rel="stylesheet" href="../lib/jquery-ui-1.10.2/css/smoothness/jquery-ui-1.10.2.min.css">
+ <!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/stats.js/17/Stats.js" crossorigin="anonymous" referrerpolicy="no-referrer"></script> -->
+
+ <!-- <script type="text/javascript" src="./webgldemodrawer.js"></script> -->
+ <script type="module" src="./drawerperformance.js"></script>
+ <style type="text/css">
+ .content{
+ max-width:960px;
+ margin: 0 auto;
+ }
+ #drawer{
+ height:800px;
+ }
+ </style>
+</head>
+<body>
+ <div class="content">
+
+ <h2>Compare performance of drawer implementations</h2>
+ <div>
+ <label>Select a drawer: </label>
+ <select id="select-drawer">
+ <option value="canvas">Canvas</option>
+ <option value="webgl">WebGL</option>
+ <option value="html">HTML</option>
+ </select>
+ <label>Num images: </label>
+ <input id="input-number" type="number" value="1" min="1" step="1">
+ <button id="create-drawer">Create drawer</button>
+ </div>
+ <div id="drawer"></div>
+
+
+ </div>
+
+
+
+</body>
+</html>
+
diff --git a/test/demo/drawerperformance.js b/test/demo/drawerperformance.js
new file mode 100644
index 00000000..25cd52e5
--- /dev/null
+++ b/test/demo/drawerperformance.js
@@ -0,0 +1,90 @@
+const sources = {
+ "rainbow":"../data/testpattern.dzi",
+ "leaves":"../data/iiif_2_0_sizes/info.json",
+ "bblue":{
+ type:'image',
+ url: "../data/BBlue.png",
+ },
+ "duomo":"https://openseadragon.github.io/example-images/duomo/duomo.dzi",
+}
+const labels = {
+ rainbow: 'Rainbow Grid',
+ leaves: 'Leaves',
+ bblue: 'Blue B',
+ duomo: 'Duomo',
+}
+let viewer;
+
+(function(){
+ var script=document.createElement('script');
+ script.onload=function(){
+ var stats=new Stats();
+ document.body.appendChild(stats.dom);
+ requestAnimationFrame(function loop(){stats.update();requestAnimationFrame(loop)});
+ };
+ script.src='https://mrdoob.github.io/stats.js/build/stats.min.js';
+ document.head.appendChild(script);
+})();
+
+
+$('#create-drawer').on('click',function(){
+ let drawerType = $('#select-drawer').val();
+ let num = Math.floor($('#input-number').val());
+
+ if(viewer){
+ viewer.destroy();
+ }
+ viewer = window.viewer = makeViewer(drawerType);
+ let tileSources = makeTileSources(num);
+
+
+
+ tileSources.forEach((ts, i) => {
+ viewer.addTiledImage({
+ tileSource: ts,
+ x: (i % 10) / 20,
+ y: Math.floor(i / 10) / 20,
+ width: 1,
+ opacity: (i % 3) === 0 ? 0.4 : 1
+ });
+ });
+
+ let movingLeft = false;
+ window.setInterval(()=>{
+ let m = movingLeft ? 1 : -1;
+ movingLeft = m === -1;
+ let dist = viewer.viewport.getBounds().width / 2 / viewer.viewport.getZoom();
+ viewer.viewport.panBy(new OpenSeadragon.Point( dist * m/2, 0));
+
+ }, 1000);
+
+});
+
+function makeViewer(drawerType){
+ let viewer = OpenSeadragon({
+ id: "drawer",
+ prefixUrl: "../../build/openseadragon/images/",
+ minZoomImageRatio:0.01,
+ maxZoomPixelRatio:100,
+ smoothTileEdgesMinZoom:1.1,
+ crossOriginPolicy: 'Anonymous',
+ ajaxWithCredentials: false,
+ drawer:drawerType,
+ blendTime:0
+ });
+
+ return viewer;
+}
+
+function makeTileSources(num){
+
+ let keys = Object.keys(sources);
+
+ let indices = Array.from(Array(num).keys());
+
+ return indices.map(index => {
+ let ts = sources[keys[index % keys.length]];
+ return ts;
+ })
+
+}
diff --git a/test/demo/webgldemodrawer.js b/test/demo/webgldemodrawer.js
index 61e7c13c..1128e4a2 100644
--- a/test/demo/webgldemodrawer.js
+++ b/test/demo/webgldemodrawer.js
@@ -1,7 +1,6 @@
// THIS CODE OVERWRITES THE ORIGINAL VERSION FOR FASTER TESTING
// i.e. it doesn't need to be re-built with grunt after every save.
-
/*
* OpenSeadragon - WebGLDrawer
*
@@ -146,11 +145,10 @@
this._TileMap = new Map();
this._gl = null;
- this._glLocs = null;
- this._glProgram = null;
- this._glUnitQuadBuffer = null;
+ this._firstPass = null;
+ this._secondPass = null;
this._glFrameBuffer = null;
- this._glTiledImageTexture = null;
+ this._renderToTexture = null;
this._glFramebufferToCanvasTransform = null;
this._outputCanvas = null;
this._outputContext = null;
@@ -200,8 +198,8 @@
});
// Delete all our created resources
- gl.deleteBuffer(this._glUnitQuadBuffer);
- gl.deleteBuffer(this._glFrameBuffer);
+ gl.deleteBuffer(this._secondPass.bufferOutputPosition);
+ gl.deleteFramebuffer(this._glFrameBuffer);
// TO DO: if/when render buffers or frame buffers are used, release them:
// gl.deleteRenderbuffer(someRenderbuffer);
// gl.deleteFramebuffer(someFramebuffer);
@@ -238,8 +236,7 @@
// Public API required by all Drawer implementations
/**
- * @returns {Boolean} returns true if canvas and webgl are supported and
- * three.js has been exposed as a global variable named THREE
+ * @returns {Boolean} returns true if canvas and webgl are supported
*/
static isSupported(){
let canvasElement = document.createElement( 'canvas' );
@@ -252,6 +249,10 @@
return !!( webglContext );
}
+ getType(){
+ return 'webgl';
+ }
+
/**
* create the HTML element (canvas in this case) that the image will be drawn into
* @returns {Element} the canvas to draw into
@@ -283,35 +284,59 @@
let rotMatrix = Mat3.makeRotation(-viewport.rotation);
let viewMatrix = scaleMatrix.multiply(rotMatrix).multiply(posMatrix);
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+
// clear the output canvas
this._outputContext.clearRect(0, 0, this._outputCanvas.width, this._outputCanvas.height);
- // TO DO: further optimization is possible.
- // If no clipping and no composite operation, the tiled images
- // can all be drawn onto the rendering canvas at the same time, avoiding
- // unnecessary clearing and copying of the pixel data.
- // For now, I'm doing it this way to replicate full functionality
- // of the context2d drawer
+ let renderingBufferHasImageData = false;
+
+ //iterate over tiled images and draw each one using a two-pass rendering pipeline if needed
+ tiledImages.forEach( (tiledImage, tiledImageIndex) => {
+
+ let useContext2dPipeline = ( tiledImage.compositeOperation ||
+ this.viewer.compositeOperation ||
+ tiledImage._clip ||
+ tiledImage._croppingPolygons ||
+ tiledImage.debugMode
+ );
+ let useTwoPassRendering = useContext2dPipeline ||(tiledImage.opacity < 1); // TO DO: check hasTransparency in addition to opacity
- //iterate over tiled imagesget the list of tiles to draw
- tiledImages.forEach( (tiledImage, i) => {
- //get the list of tiles to draw
let tilesToDraw = tiledImage.getTilesToDraw();
if(tilesToDraw.length === 0){
return;
}
- // bind to the framebuffer for render-to-texture
- gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
+ // using the context2d pipeline requires a clean rendering (back) buffer to start
+ if(useContext2dPipeline){
+ // if the rendering buffer has image data currently, write it to the output canvas now and clear it
- // clear the buffer
- gl.clear(gl.COLOR_BUFFER_BIT);
+ if(renderingBufferHasImageData){
+ this._outputContext.drawImage(this._renderingCanvas, 0, 0);
+ }
+
+ // clear the buffer
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+ }
+
+ // First rendering pass: compose tiles that make up this tiledImage
+ gl.useProgram(this._firstPass.shaderProgram);
+
+ // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null)
+ if(useTwoPassRendering){
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
+ // clear the buffer to draw a new image
+ gl.clear(gl.COLOR_BUFFER_BIT);
+ } else {
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ // no need to clear, just draw on top of the existing pixels
+ }
- // set opacity for this image
- gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity);
@@ -330,62 +355,142 @@
overallMatrix = viewMatrix.multiply(localMatrix);
}
- // iterate over tiles and draw each one to the buffer
- for(let ti = 0; ti < tilesToDraw.length; ti++){
- let tile = tilesToDraw[ti].tile;
- let textureInfo = this._TextureMap.get(tile.getCanvasContext().canvas);
+ let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS);
+ let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex
+ let textureDataArray = new Array(maxTextures);
+ let matrixArray = new Array(maxTextures);
+ let opacityArray = new Array(maxTextures);
+
+ // iterate over tiles and add data for each one to the buffers
+ for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){
+ let tile = tilesToDraw[tileIndex].tile;
+ let index = tileIndex % maxTextures;
+ let tileContext = tile.getCanvasContext();
+
+ let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null;
if(textureInfo){
- this._drawTile(tile, tiledImage, textureInfo, overallMatrix, tiledImage.opacity);
+ this._getTileData(tile, tiledImage, textureInfo, overallMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray);
} else {
// console.log('No tile info', tile);
}
- }
+ if( (index === maxTextures - 1) || (tileIndex === tilesToDraw.length - 1)){
+ // We've filled up the buffers: time to draw this set of tiles
+ // bind each tile's texture to the appropriate gl.TEXTURE#
+ for(let i = 0; i <= index; i++){
+ gl.activeTexture(gl.TEXTURE0 + i);
+ gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]);
+ }
- // Draw from the Framebuffer onto the rendering canvas buffer
+ // set the buffer data for the texture coordinates to use for each tile
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
+ gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW);
- gl.flush(); // finish drawing to the texture
- gl.bindFramebuffer(gl.FRAMEBUFFER, null); // null means bind to the backbuffer for drawing
- gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); // bind the rendered texture to use
- gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+ // set the transform matrix uniform for each tile
+ matrixArray.forEach( (matrix, index) => {
+ gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix);
+ });
+ // set the opacity uniform for each tile
+ gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray));
- // set up the matrix to draw the whole framebuffer to the entire clip space
- gl.uniformMatrix3fv(this._glLocs.uMatrix, false, this._glFramebufferToCanvasTransform);
+ // bind vertex buffers and (re)set attributes before calling gl.drawArrays()
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition);
+ gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
- // reset texturebuffer to unit quad
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW);
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
+ gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
- // set opacity to the value for the current tiledImage
- this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity);
- gl.drawArrays(gl.TRIANGLES, 0, 6);
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex);
+ gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0);
- // iterate over any filters - filters can use this._glTiledImageTexture to get rendered data if desired
- let filters = this.filters || [];
- for(let fi = 0; fi < filters.length; fi++){
- let filter = this.filters[fi];
- if(filter.createProgram && !filter.program){
- filter.createProgram(gl);
- }
- if(filter.apply){
- filter.apply(gl, this._glTiledImageTexture, tiledImage); // filter.apply should write data on top of the backbuffer (bound above)
+ // Draw! 6 vertices per tile (2 triangles per rectangle)
+ gl.drawArrays(gl.TRIANGLES, 0, 6 * (index + 1) );
}
}
- gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary?
+ // gl.flush(); // is this necessary?
- // draw from the rendering canvas onto the output canvas, clipping/cropping if needed.
- this._renderToOutputCanvas(tiledImage, tilesToDraw, i);
+ if(useTwoPassRendering){
+ // Second rendering pass: Render the tiled image from the framebuffer into the back buffer
+ gl.useProgram(this._secondPass.shaderProgram);
+
+ // set the rendering target to the back buffer (null)
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+
+ // bind the rendered texture from the first pass to use during this second pass
+ gl.activeTexture(gl.TEXTURE0);
+ gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
+
+ // set opacity to the value for the current tiledImage
+ this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity);
+
+ // bind buffers and set attributes before calling gl.drawArrays
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition);
+ gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition);
+ gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
+
+ // Draw the quad (two triangles)
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
+
+ // TO DO: is this the mechanism we want to use here?
+ // iterate over any filters - filters can use this._renderToTexture to get rendered data if desired
+ let filters = this.filters || [];
+ for(let fi = 0; fi < filters.length; fi++){
+ let filter = this.filters[fi];
+ if(filter.apply){
+ filter.apply(gl); // filter.apply should write data on top of the backbuffer (bound above)
+ }
+ }
+ }
+
+ renderingBufferHasImageData = true;
+
+ // gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary?
+
+ if(useContext2dPipeline){
+ // draw from the rendering canvas onto the output canvas, clipping/cropping if needed.
+ this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex);
+ renderingBufferHasImageData = false;
+ // clear the buffer
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
+ }
+
+ // Fire tiled-image-drawn event.
+ // TO DO: the image data may not be on the output canvas yet!!
+ if( this.viewer ){
+ /**
+ * Raised when a tiled image is drawn to the canvas. Only valid
+ * for webgl drawer.
+ *
+ * @event tiled-image-drawn
+ * @memberof OpenSeadragon.Viewer
+ * @type {object}
+ * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event.
+ * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn.
+ * @property {Array} tiles - An array of Tile objects that were drawn.
+ * @property {?Object} userData - Arbitrary subscriber-defined object.
+ */
+ this.viewer.raiseEvent( 'tiled-image-drawn', {
+ tiledImage: tiledImage,
+ tiles: tilesToDraw.map(info => info.tile),
+ });
+ }
});
+ // TO DO: the line below is a test!
+ if(renderingBufferHasImageData){
+ this._outputContext.drawImage(this._renderingCanvas, 0, 0);
+ }
}
// Public API required by all Drawer implementations
/**
- * Set the context2d imageSmoothingEnabled parameter
- * @param {Boolean} enabled
- */
+ * Set the context2d imageSmoothingEnabled parameter
+ * @param {Boolean} enabled
+ */
setImageSmoothingEnabled(enabled){
this._clippingContext.imageSmoothingEnabled = enabled;
this._outputContext.imageSmoothingEnabled = enabled;
@@ -417,13 +522,13 @@
}
/**
- * Draw data from the rendering canvas onto the output canvas, with clipping,
- * cropping and/or debug info as requested.
- * @private
- * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw
- * @param {Array} tilesToDraw - array of objects containing tiles that were drawn
- */
- _renderToOutputCanvas(tiledImage, tilesToDraw, tiledImageIndex){
+ * Draw data from the rendering canvas onto the output canvas, with clipping,
+ * cropping and/or debug info as requested.
+ * @private
+ * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw
+ * @param {Array} tilesToDraw - array of objects containing tiles that were drawn
+ */
+ _applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex){
// composite onto the output canvas, clipping if necessary
this._outputContext.save();
@@ -444,39 +549,19 @@
this._drawDebugInfo(tilesToDraw, tiledImage, strokeStyle, fillStyle);
}
- // Fire tiled-image-drawn event now that the data is on the output canvas
- if( this.viewer ){
- /**
- * Raised when a tiled image is drawn to the canvas. Only valid
- * for webgl drawer.
- *
- * @event tiled-image-drawn
- * @memberof OpenSeadragon.Viewer
- * @type {object}
- * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event.
- * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn.
- * @property {Array} tiles - An array of Tile objects that were drawn.
- * @property {?Object} userData - Arbitrary subscriber-defined object.
- */
- this.viewer.raiseEvent( 'tiled-image-drawn', {
- tiledImage: tiledImage,
- tiles: tilesToDraw.map(info => info.tile),
- });
- }
+
}
// private
- _drawTile(tile, tiledImage, textureInfo, viewMatrix, imageOpacity){
+ _getTileData(tile, tiledImage, textureInfo, viewMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray){
- let gl = this._gl;
let texture = textureInfo.texture;
let textureQuad = textureInfo.position;
- // set the vertices into the non-overlapped portion of the texture
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, textureQuad, gl.DYNAMIC_DRAW);
+ // set the position of this texture
+ texturePositionArray.set(textureQuad, index * 12);
- // compute offsets for overlap
+ // compute offsets that account for tile overlap; needed for calculating the transform matrix appropriately
let overlapFraction = this._calculateOverlapFraction(tile, tiledImage);
let xOffset = tile.positionedBounds.width * overlapFraction.x;
let yOffset = tile.positionedBounds.height * overlapFraction.y;
@@ -507,41 +592,164 @@
let overallMatrix = viewMatrix.multiply(matrix);
- // set opacity for this image
- this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tile.opacity); // imageOpacity *
-
- gl.uniformMatrix3fv(this._glLocs.uMatrix, false, overallMatrix.values);
- gl.bindTexture(gl.TEXTURE_2D, texture);
+ opacityArray[index] = tile.opacity;// * tiledImage.opacity;
+ textureDataArray[index] = texture;
+ matrixArray[index] = overallMatrix.values;
if(this.continuousTileRefresh){
- // Upload the image into the texture (already bound to TEXTURE_2D above)
+ // Upload the image into the texture
+ // TO DO: test if this works appropriately
let tileContext = tile.getCanvasContext();
this._raiseTileDrawingEvent(tiledImage, this._outputContext, tile, tileContext);
this._uploadImageData(tileContext, tile, tiledImage);
}
-
- gl.drawArrays(gl.TRIANGLES, 0, 6);
}
_setupRenderer(){
-
- if(!this._gl){
+ let gl = this._gl;
+ if(!gl){
$.console.error('_setupCanvases must be called before _setupRenderer');
}
+ this._unitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1); // used a few places; create once and store the result
+ this._makeFirstPassShaderProgram();
+ this._makeSecondPassShaderProgram();
+
+ // set up the texture to render to in the first pass, and which will be used for rendering the second pass
+ this._renderToTexture = gl.createTexture();
+ gl.activeTexture(gl.TEXTURE0);
+ gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+ // set up the framebuffer for render-to-texture
+ this._glFrameBuffer = gl.createFramebuffer();
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
+ gl.framebufferTexture2D(
+ gl.FRAMEBUFFER,
+ gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0
+ gl.TEXTURE_2D, // attach a 2D texture
+ this._renderToTexture, // the texture to attach
+ 0
+ );
+
+ gl.enable(gl.BLEND);
+ gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
+
+ }
+
+ _makeFirstPassShaderProgram(){
+ let numTextures = this._glNumTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS);
+ let makeMatrixUniforms = () => {
+ return [...Array(numTextures).keys()].map(index => `uniform mat3 u_matrix_${index};`).join('\n');
+ };
+ let makeConditionals = () => {
+ return [...Array(numTextures).keys()].map(index => `${index > 0 ? 'else ' : ''}if(int(a_index) == ${index}) { transform_matrix = u_matrix_${index}; }`).join('\n');
+ };
+
+ const vertexShaderProgram = `
+ attribute vec2 a_output_position;
+ attribute vec2 a_texture_position;
+ attribute float a_index;
+
+ ${makeMatrixUniforms()} // create a uniform mat3 for each potential tile to draw
+
+ varying vec2 v_texture_position;
+ varying float v_image_index;
+
+ void main() {
+
+ mat3 transform_matrix; // value will be set by the if/elses in makeConditional()
+
+ ${makeConditionals()}
+
+ gl_Position = vec4(transform_matrix * vec3(a_output_position, 1), 1);
+
+ v_texture_position = a_texture_position;
+ v_image_index = a_index;
+ }
+ `;
+
+ const fragmentShaderProgram = `
+ precision mediump float;
+
+ // our textures
+ uniform sampler2D u_images[${numTextures}];
+ // our opacities
+ uniform float u_opacities[${numTextures}];
+
+ // the varyings passed in from the vertex shader.
+ varying vec2 v_texture_position;
+ varying float v_image_index;
+
+ void main() {
+ // can't index directly with a variable, need to use a loop iterator hack
+ for(int i = 0; i < ${numTextures}; ++i){
+ if(i == int(v_image_index)){
+ gl_FragColor = texture2D(u_images[i], v_texture_position) * u_opacities[i];
+ }
+ }
+ }
+ `;
+
+ let gl = this._gl;
+
+ let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram);
+ gl.useProgram(program);
+
+ // get locations of attributes and uniforms, and create buffers for each attribute
+ this._firstPass = {
+ shaderProgram: program,
+ aOutputPosition: gl.getAttribLocation(program, 'a_output_position'),
+ aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'),
+ aIndex: gl.getAttribLocation(program, 'a_index'),
+ uTransformMatrices: [...Array(this._glNumTextures).keys()].map(i=>gl.getUniformLocation(program, `u_matrix_${i}`)),
+ uImages: gl.getUniformLocation(program, 'u_images'),
+ uOpacities: gl.getUniformLocation(program, 'u_opacities'),
+ bufferOutputPosition: gl.createBuffer(),
+ bufferTexturePosition: gl.createBuffer(),
+ bufferIndex: gl.createBuffer(),
+ };
+
+ gl.uniform1iv(this._firstPass.uImages, [...Array(numTextures).keys()]);
+
+ // provide coordinates for the rectangle in output space, i.e. a unit quad for each one.
+ let outputQuads = new Float32Array(numTextures * 12);
+ for(let i = 0; i < numTextures; ++i){
+ outputQuads.set(Float32Array.from(this._unitQuad), i * 12);
+ }
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition);
+ gl.bufferData(gl.ARRAY_BUFFER, outputQuads, gl.STATIC_DRAW); // bind data statically here, since it's unchanging
+ gl.enableVertexAttribArray(this._firstPass.aOutputPosition);
+
+ // provide texture coordinates for the rectangle in image (texture) space. Data will be set later.
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
+ gl.enableVertexAttribArray(this._firstPass.aTexturePosition);
+
+ // for each vertex, provide an index into the array of textures/matrices to use for the correct tile
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex);
+ let indices = [...Array(this._glNumTextures).keys()].map(i => Array(6).fill(i)).flat(); // repeat each index 6 times, for the 6 vertices per tile (2 triangles)
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(indices), gl.STATIC_DRAW); // bind data statically here, since it's unchanging
+ gl.enableVertexAttribArray(this._firstPass.aIndex);
+
+ }
+
+ _makeSecondPassShaderProgram(){
const vertexShaderProgram = `
attribute vec2 a_output_position;
attribute vec2 a_texture_position;
uniform mat3 u_matrix;
- varying vec2 v_texCoord;
+ varying vec2 v_texture_position;
void main() {
- gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1);
+ gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1);
- v_texCoord = a_texture_position;
+ v_texture_position = a_texture_position;
}
`;
@@ -552,66 +760,48 @@
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
- varying vec2 v_texCoord;
+ varying vec2 v_texture_position;
// the opacity multiplier for the image
uniform float u_opacity_multiplier;
void main() {
- gl_FragColor = texture2D(u_image, v_texCoord);
- gl_FragColor *= u_opacity_multiplier;
+ gl_FragColor = texture2D(u_image, v_texture_position);
+ gl_FragColor *= u_opacity_multiplier;
}
`;
let gl = this._gl;
- this._glProgram = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram);
- gl.useProgram(this._glProgram);
- gl.enable(gl.BLEND);
- gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
- this._glLocs = {
- aOutputPosition: gl.getAttribLocation(this._glProgram, 'a_output_position'),
- aTexturePosition: gl.getAttribLocation(this._glProgram, 'a_texture_position'),
- uMatrix: gl.getUniformLocation(this._glProgram, 'u_matrix'),
- uImage: gl.getUniformLocation(this._glProgram, 'u_image'),
- uOpacityMultiplier: gl.getUniformLocation(this._glProgram, 'u_opacity_multiplier')
+ let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram);
+ gl.useProgram(program);
+
+ // get locations of attributes and uniforms, and create buffers for each attribute
+ this._secondPass = {
+ shaderProgram: program,
+ aOutputPosition: gl.getAttribLocation(program, 'a_output_position'),
+ aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'),
+ uMatrix: gl.getUniformLocation(program, 'u_matrix'),
+ uImage: gl.getUniformLocation(program, 'u_image'),
+ uOpacityMultiplier: gl.getUniformLocation(program, 'u_opacity_multiplier'),
+ bufferOutputPosition: gl.createBuffer(),
+ bufferTexturePosition: gl.createBuffer(),
};
- this._glUnitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1);
- // provide texture coordinates for the rectangle in output space.
- this._glUnitQuadBuffer = gl.createBuffer(); //keep reference to clear it later
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glUnitQuadBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.STATIC_DRAW);
- gl.enableVertexAttribArray(this._glLocs.aOutputPosition);
- gl.vertexAttribPointer(this._glLocs.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
+
+ // provide coordinates for the rectangle in output space, i.e. a unit quad for each one.
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition);
+ gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.STATIC_DRAW); // bind data statically here since it's unchanging
+ gl.enableVertexAttribArray(this._secondPass.aOutputPosition);
// provide texture coordinates for the rectangle in image (texture) space.
- this._glTextureBuffer = gl.createBuffer(); //keep reference to clear it later
- gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer);
- gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW); // use unit quad to start, will be updated per tile
- gl.enableVertexAttribArray(this._glLocs.aTexturePosition);
- gl.vertexAttribPointer(this._glLocs.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
-
- // setup the framebuffer
- this._glTiledImageTexture = gl.createTexture();
- gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture);
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
- this._glFrameBuffer = gl.createFramebuffer();
- gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
- gl.framebufferTexture2D(
- gl.FRAMEBUFFER,
- gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0
- gl.TEXTURE_2D, // attach a 2D texture
- this._glTiledImageTexture, // the texture to attach
- 0
- );
-
- this._glFramebufferToCanvasTransform = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5)).values;
-
+ gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition);
+ gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.DYNAMIC_DRAW); // bind data statically here since it's unchanging
+ gl.enableVertexAttribArray(this._secondPass.aTexturePosition);
+ // set the matrix that transforms the framebuffer to clip space
+ let matrix = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5));
+ gl.uniformMatrix3fv(this._secondPass.uMatrix, false, matrix.values);
}
_resizeRenderer(){
@@ -621,10 +811,11 @@
gl.viewport(0, 0, w, h);
//release the old texture
- gl.deleteTexture(this._glTiledImageTexture);
+ gl.deleteTexture(this._renderToTexture);
//create a new texture and set it up
- this._glTiledImageTexture = gl.createTexture();
- gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture);
+ this._renderToTexture = gl.createTexture();
+ gl.activeTexture(gl.TEXTURE0);
+ gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, w, h, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
@@ -632,7 +823,7 @@
//bind the frame buffer to the new texture
gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._glTiledImageTexture, 0);
+ gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderToTexture, 0);
}
@@ -714,7 +905,7 @@
position = this._makeQuadVertexBuffer(left, right, top, bottom);
} else {
// no overlap: this texture can use the unit quad as it's position data
- position = this._glUnitQuad;
+ position = this._unitQuad;
}
let textureInfo = {
@@ -724,7 +915,7 @@
// add it to our _TextureMap
this._TextureMap.set(canvas, textureInfo);
-
+ gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
// Set the parameters so we can render any size image.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
@@ -758,9 +949,17 @@
let gl = this._gl;
let canvas = tileContext.canvas;
- // This depends on gl.TEXTURE_2D being bound to the texture
- // associated with this canvas before calling this function
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas);
+ try{
+ if(!canvas){
+ throw('Tile context does not have a canvas', tileContext);
+ }
+ // This depends on gl.TEXTURE_2D being bound to the texture
+ // associated with this canvas before calling this function
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas);
+ } catch (e){
+ $.console.error('Error uploading image data to WebGL', e);
+ }
+
}
@@ -772,13 +971,13 @@
_cleanupImageData(tileCanvas){
let textureInfo = this._TextureMap.get(tileCanvas);
//remove from the map
+ this._TextureMap.delete(tileCanvas);
+
//release the texture from the GPU
if(textureInfo){
this._gl.deleteTexture(textureInfo.texture);
}
- //release the texture from the GPU
- this._gl.deleteTexture(textureInfo.texture);
// release the position buffer from the GPU
// TO DO: do this!
}
@@ -1033,14 +1232,6 @@
};
- // $.WebGLDrawer.Filter = class{
- // constructor(gl){
- // this.gl = gl;
- // }
- // apply(){
-
- // }
- // };
}( OpenSeadragon ));