From fcf20be8ea2a862d0c29f1523061c4c7a9aa45c7 Mon Sep 17 00:00:00 2001 From: Aiosa <469130@mail.muni.cz> Date: Sun, 4 Feb 2024 18:48:25 +0100 Subject: [PATCH] Drawers now use new cache API to draw onto a canvas. The type conversion now requires also the tile argument so that conversion can rely on the tile metadata. --- src/canvasdrawer.js | 112 +++++++++----------- src/datatypeconvertor.js | 67 ++++++++---- src/drawerbase.js | 55 +++++++++- src/htmldrawer.js | 4 +- src/tile.js | 9 +- src/tilecache.js | 31 ++++-- src/tiledimage.js | 7 +- src/viewer.js | 4 +- src/webgldrawer.js | 178 ++++++++++++++------------------ test/modules/basic.js | 28 ++--- test/modules/tilecache.js | 20 ++-- test/modules/type-conversion.js | 31 +++--- 12 files changed, 300 insertions(+), 246 deletions(-) diff --git a/src/canvasdrawer.js b/src/canvasdrawer.js index d4ec6231..086431af 100644 --- a/src/canvasdrawer.js +++ b/src/canvasdrawer.js @@ -50,6 +50,8 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ constructor(options){ super(options); + this.declareSupportedDataFormats("context2d"); + /** * The HTML element (canvas) that this drawer uses for drawing * @member {Element} canvas @@ -255,26 +257,26 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ * */ _drawTiles( tiledImage ) { - var lastDrawn = tiledImage.getTilesToDraw().map(info => info.tile); + const lastDrawn = tiledImage.getTilesToDraw().map(info => info.tile); if (tiledImage.opacity === 0 || (lastDrawn.length === 0 && !tiledImage.placeholderFillStyle)) { return; } - var tile = lastDrawn[0]; - var useSketch; + let tile = lastDrawn[0]; + let useSketch; if (tile) { useSketch = tiledImage.opacity < 1 || (tiledImage.compositeOperation && tiledImage.compositeOperation !== 'source-over') || (!tiledImage._isBottomItem() && - tiledImage.source.hasTransparency(tile.context2D, tile.getUrl(), tile.ajaxHeaders, tile.postData)); + tiledImage.source.hasTransparency(null, tile.getUrl(), tile.ajaxHeaders, tile.postData)); } - var sketchScale; - var sketchTranslate; + let sketchScale; + let sketchTranslate; - var zoom = this.viewport.getZoom(true); - var imageZoom = tiledImage.viewportToImageZoom(zoom); + const zoom = this.viewport.getZoom(true); + const imageZoom = tiledImage.viewportToImageZoom(zoom); if (lastDrawn.length > 1 && imageZoom > tiledImage.smoothTileEdgesMinZoom && @@ -284,13 +286,19 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ // So we have to composite them at ~100% and scale them up together. // Note: Disabled on iOS devices per default as it causes a native crash useSketch = true; - sketchScale = tile.getScaleForEdgeSmoothing(); + + const context = tile.length && this.getCompatibleData(tile); + if (context) { + sketchScale = context.canvas.width / (tile.size.x * $.pixelDensityRatio); + } else { + sketchScale = 1; + } sketchTranslate = tile.getTranslationForEdgeSmoothing(sketchScale, this._getCanvasSize(false), this._getCanvasSize(true)); } - var bounds; + let bounds; if (useSketch) { if (!sketchScale) { // Except when edge smoothing, we only clean the part of the @@ -337,13 +345,13 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ } } - var usedClip = false; + let usedClip = false; if ( tiledImage._clip ) { this._saveContext(useSketch); - var box = tiledImage.imageToViewportRectangle(tiledImage._clip, true); + let box = tiledImage.imageToViewportRectangle(tiledImage._clip, true); box = box.rotate(-tiledImage.getRotation(true), tiledImage._getRotationPoint(true)); - var clipRect = this.viewportToDrawerRectangle(box); + let clipRect = this.viewportToDrawerRectangle(box); if (sketchScale) { clipRect = clipRect.times(sketchScale); } @@ -356,17 +364,17 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ } if (tiledImage._croppingPolygons) { - var self = this; + const self = this; if(!usedClip){ this._saveContext(useSketch); } try { - var polygons = tiledImage._croppingPolygons.map(function (polygon) { + const polygons = tiledImage._croppingPolygons.map(function (polygon) { return polygon.map(function (coord) { - var point = tiledImage + const point = tiledImage .imageToViewportCoordinates(coord.x, coord.y, true) .rotate(-tiledImage.getRotation(true), tiledImage._getRotationPoint(true)); - var clipPoint = self.viewportCoordToDrawerCoord(point); + let clipPoint = self.viewportCoordToDrawerCoord(point); if (sketchScale) { clipPoint = clipPoint.times(sketchScale); } @@ -384,7 +392,7 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ } if ( tiledImage.placeholderFillStyle && tiledImage._hasOpaqueTile === false ) { - var placeholderRect = this.viewportToDrawerRectangle(tiledImage.getBounds(true)); + let placeholderRect = this.viewportToDrawerRectangle(tiledImage.getBounds(true)); if (sketchScale) { placeholderRect = placeholderRect.times(sketchScale); } @@ -392,7 +400,7 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ placeholderRect = placeholderRect.translate(sketchTranslate); } - var fillStyle = null; + let fillStyle; if ( typeof tiledImage.placeholderFillStyle === "function" ) { fillStyle = tiledImage.placeholderFillStyle(tiledImage, this.context); } @@ -403,19 +411,18 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ this._drawRectangle(placeholderRect, fillStyle, useSketch); } - var subPixelRoundingRule = determineSubPixelRoundingRule(tiledImage.subPixelRoundingForTransparency); + const subPixelRoundingRule = determineSubPixelRoundingRule(tiledImage.subPixelRoundingForTransparency); - var shouldRoundPositionAndSize = false; + let shouldRoundPositionAndSize = false; if (subPixelRoundingRule === $.SUBPIXEL_ROUNDING_OCCURRENCES.ALWAYS) { shouldRoundPositionAndSize = true; } else if (subPixelRoundingRule === $.SUBPIXEL_ROUNDING_OCCURRENCES.ONLY_AT_REST) { - var isAnimating = this.viewer && this.viewer.isAnimating(); - shouldRoundPositionAndSize = !isAnimating; + shouldRoundPositionAndSize = !(this.viewer && this.viewer.isAnimating()); } // Iterate over the tiles to draw, and draw them - for (var i = 0; i < lastDrawn.length; i++) { + for (let i = 0; i < lastDrawn.length; i++) { tile = lastDrawn[ i ]; this._drawTile( tile, tiledImage, useSketch, sketchScale, sketchTranslate, shouldRoundPositionAndSize, tiledImage.source ); @@ -499,9 +506,7 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ this._drawDebugInfo( tiledImage, lastDrawn ); // Fire tiled-image-drawn event. - this._raiseTiledImageDrawnEvent(tiledImage, lastDrawn); - } /** @@ -559,52 +564,25 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ $.console.assert(tile, '[Drawer._drawTile] tile is required'); $.console.assert(tiledImage, '[Drawer._drawTile] drawingHandler is required'); - var context = this._getContext(useSketch); - scale = scale || 1; - this._drawTileToCanvas(tile, context, tiledImage, scale, translate, shouldRoundPositionAndSize, source); - - } - - /** - * Renders the tile in a canvas-based context. - * @private - * @function - * @param {OpenSeadragon.Tile} tile - the tile to draw to the canvas - * @param {Canvas} context - * @param {OpenSeadragon.TiledImage} tiledImage - Method for firing the drawing event. - * drawingHandler({context, tile, rendered}) - * where rendered is the context with the pre-drawn image. - * @param {Number} [scale=1] - Apply a scale to position and size - * @param {OpenSeadragon.Point} [translate] - A translation vector - * @param {Boolean} [shouldRoundPositionAndSize] - Tells whether to round - * position and size of tiles supporting alpha channel in non-transparency - * context. - * @param {OpenSeadragon.TileSource} source - The source specification of the tile. - */ - _drawTileToCanvas( tile, context, tiledImage, scale, translate, shouldRoundPositionAndSize, source) { - - var position = tile.position.times($.pixelDensityRatio), - size = tile.size.times($.pixelDensityRatio), - rendered; - - if (!tile.context2D && !tile.cacheImageRecord) { - $.console.warn( - '[Drawer._drawTileToCanvas] attempting to draw tile %s when it\'s not cached', - tile.toString()); - return; - } - - rendered = tile.getCanvasContext(); - - if ( !tile.loaded || !rendered ){ + if ( !tile.loaded ){ $.console.warn( "Attempting to draw tile %s when it's not yet loaded.", tile.toString() ); - return; } + const rendered = this.getCompatibleData(tile); + if (!rendered) { + return; + } + + const context = this._getContext(useSketch); + scale = scale || 1; + + let position = tile.position.times($.pixelDensityRatio), + size = tile.size.times($.pixelDensityRatio); + context.save(); // context.globalAlpha = this.options.opacity; // this was deprecated previously and should not be applied as it is set per TiledImage @@ -644,7 +622,7 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ this._raiseTileDrawingEvent(tiledImage, context, tile, rendered); - var sourceWidth, sourceHeight; + let sourceWidth, sourceHeight; if (tile.sourceBounds) { sourceWidth = Math.min(tile.sourceBounds.width, rendered.canvas.width); sourceHeight = Math.min(tile.sourceBounds.height, rendered.canvas.height); @@ -672,6 +650,8 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ context.restore(); } + + /** * Get the context of the main or sketch canvas * @private diff --git a/src/datatypeconvertor.js b/src/datatypeconvertor.js index 2cc6c2da..31a5a450 100644 --- a/src/datatypeconvertor.js +++ b/src/datatypeconvertor.js @@ -148,13 +148,24 @@ class WeightedGraph { } /** - * Node on the conversion path in OpenSeadragon.converter.getConversionPath(). + * Edge.transform function on the conversion path in OpenSeadragon.converter.getConversionPath(). * It can be also conversion to undefined if used as destructor implementation. * * @callback TypeConvertor * @memberof OpenSeadragon - * @param {?} data data in the input format - * @return {?} data in the output format + * @param {OpenSeadragon.Tile} tile reference tile that owns the data + * @param {any} data data in the input format + * @returns {any} data in the output format + */ + +/** + * Destructor called every time a data type is to be destroyed or converted to another type. + * + * @callback TypeDestructor + * @memberof OpenSeadragon + * @param {any} data data in the format the destructor is registered for + * @returns {any} can return any value that is carried over to the caller if desirable. + * Note: not used by the OSD cache system. */ /** @@ -184,13 +195,13 @@ $.DataTypeConvertor = class { this.copyings = {}; // Teaching OpenSeadragon built-in conversions: - const imageCreator = (url) => new $.Promise((resolve, reject) => { + const imageCreator = (tile, url) => new $.Promise((resolve, reject) => { const img = new Image(); img.onerror = img.onabort = reject; img.onload = () => resolve(img); img.src = url; }); - const canvasContextCreator = (imageData) => { + const canvasContextCreator = (tile, imageData) => { const canvas = document.createElement( 'canvas' ); canvas.width = imageData.width; canvas.height = imageData.height; @@ -199,15 +210,25 @@ $.DataTypeConvertor = class { return context; }; - this.learn("context2d", "url", ctx => ctx.canvas.toDataURL(), 1, 2); - this.learn("image", "url", image => image.url); + this.learn("context2d", "url", (tile, ctx) => ctx.canvas.toDataURL(), 1, 2); + this.learn("image", "url", (tile, image) => image.url); this.learn("image", "context2d", canvasContextCreator, 1, 1); this.learn("url", "image", imageCreator, 1, 1); //Copies - this.learn("image", "image", image => imageCreator(image.src), 1, 1); - this.learn("url", "url", url => url, 0, 1); //strings are immutable, no need to copy - this.learn("context2d", "context2d", ctx => canvasContextCreator(ctx.canvas)); + this.learn("image", "image", (tile, image) => imageCreator(tile, image.src), 1, 1); + this.learn("url", "url", (tile, url) => url, 0, 1); //strings are immutable, no need to copy + this.learn("context2d", "context2d", (tile, ctx) => canvasContextCreator(tile, ctx.canvas)); + + /** + * Free up canvas memory + * (iOS 12 or higher on 2GB RAM device has only 224MB canvas memory, + * and Safari keeps canvas until its height and width will be set to 0). + */ + this.learnDestroy("context2d", ctx => { + ctx.canvas.width = 0; + ctx.canvas.height = 0; + }); } /** @@ -263,9 +284,9 @@ $.DataTypeConvertor = class { * Teach the system to convert data type 'from' -> 'to' * @param {string} from unique ID of the data item 'from' * @param {string} to unique ID of the data item 'to' - * @param {OpenSeadragon.TypeConvertor} callback convertor that takes type 'from', and converts to type 'to'. - * Callback can return function. This function returns the data in type 'to', - * it can return also the value wrapped in a Promise (returned in resolve) or it can be async function. + * @param {OpenSeadragon.TypeConvertor} callback convertor that takes two arguments: a tile reference, and + * a data object of a type 'from'; and converts this data object to type 'to'. It can return also the value + * wrapped in a Promise (returned in resolve) or it can be async function. * @param {Number} [costPower=0] positive cost class of the conversion, smaller or equal than 7. * Should reflect the actual cost of the conversion: * - if nothing must be done and only reference is retrieved (or a constant operation done), @@ -298,7 +319,7 @@ $.DataTypeConvertor = class { * for example, textures loaded to GPU have to be also manually removed when not needed anymore. * Needs to be defined only when the created object has extra deletion process. * @param {string} type - * @param {OpenSeadragon.TypeConvertor} callback destructor, receives the object created, + * @param {OpenSeadragon.TypeDestructor} callback destructor, receives the object created, * it is basically a type conversion to 'undefined' - thus the type. */ learnDestroy(type, callback) { @@ -312,12 +333,13 @@ $.DataTypeConvertor = class { * Note: conversion DOES NOT COPY data if [to] contains type 'from' (e.g., the cheapest conversion is no conversion). * It automatically calls destructor on immediate types, but NOT on the x and the result. You should call these * manually if these should be destroyed. - * @param {*} x data item to convert + * @param {OpenSeadragon.Tile} tile + * @param {any} data data item to convert * @param {string} from data item type * @param {string} to desired type(s) * @return {OpenSeadragon.Promise} promise resolution with type 'to' or undefined if the conversion failed */ - convert(x, from, ...to) { + convert(tile, data, from, ...to) { const conversionPath = this.getConversionPath(from, to); if (!conversionPath) { $.console.error(`[OpenSeadragon.convertor.convert] Conversion conversion ${from} ---> ${to} cannot be done!`); @@ -331,7 +353,7 @@ $.DataTypeConvertor = class { return $.Promise.resolve(x); } let edge = conversionPath[i]; - let y = edge.transform(x); + let y = edge.transform(tile, x); if (!y) { $.console.warn(`[OpenSeadragon.convertor.convert] data mid result falsey value (while converting to %s)`, edge.target); return $.Promise.resolve(); @@ -344,19 +366,20 @@ $.DataTypeConvertor = class { return result.then(res => step(res, i + 1)); }; //destroy only mid-results, but not the original value - return step(x, 0, false); + return step(data, 0, false); } /** * Destroy the data item given. + * @param {OpenSeadragon.Tile} tile + * @param {any} data data item to convert * @param {string} type data type - * @param {?} data * @return {OpenSeadragon.Promise|undefined} promise resolution with data passed from constructor */ - copy(data, type) { + copy(tile, data, type) { const copyTransform = this.copyings[type]; if (copyTransform) { - const y = copyTransform(data); + const y = copyTransform(tile, data); return $.type(y) === "promise" ? y : $.Promise.resolve(y); } $.console.warn(`[OpenSeadragon.convertor.copy] is not supported with type %s`, type); @@ -399,7 +422,7 @@ $.DataTypeConvertor = class { } if (Array.isArray(to)) { - $.console.assert(typeof to === "string" || to.length > 0, "[getConversionPath] conversion 'to' type must be defined."); + $.console.assert(to.length > 0, "[getConversionPath] conversion 'to' type must be defined."); let bestCost = Infinity; //FIXME: pre-compute all paths in 'to' array? could be efficient for multiple diff --git a/src/drawerbase.js b/src/drawerbase.js index 29d7a3b4..19317d76 100644 --- a/src/drawerbase.js +++ b/src/drawerbase.js @@ -77,7 +77,7 @@ OpenSeadragon.DrawerBase = class DrawerBase{ this.container.style.textAlign = "left"; this.container.appendChild( this.canvas ); - this._checkForAPIOverrides(); + this._checkInterfaceImplementation(); } // protect the canvas member with a getter @@ -98,6 +98,54 @@ OpenSeadragon.DrawerBase = class DrawerBase{ return undefined; } + /** + * Define which data types are compatible for this drawer to work with. + * See default type list in OpenSeadragon.DataTypeConvertor + * @param formats + */ + declareSupportedDataFormats(...formats) { + this._formats = formats; + } + + /** + * Retrieve data types + * @return {[string]} + */ + getSupportedDataFormats() { + if (!this._formats || this._formats.length < 1) { + $.console.error("A drawer must define its supported rendering data types using declareSupportedDataFormats!"); + } + return this._formats; + } + + /** + * Check a particular cache record is compatible. + * This function _MUST_ be called: if it returns a falsey + * value, the rendering _MUST NOT_ proceed. It should + * await next animation frames and check again for availability. + * @param {OpenSeadragon.Tile} tile + */ + getCompatibleData(tile) { + const cache = tile.getCache(tile.cacheKey); + if (!cache) { + return null; + } + + const formats = this.getSupportedDataFormats(); + if (!formats.includes(cache.type)) { + cache.transformTo(formats.length > 1 ? formats : formats[0]); + return false; // type is NOT compatible + } + + // Cache in the process of loading, no-op + if (!cache.loaded) { + return false; // cache is NOT ready + } + + // Ensured compatible + return cache.data; + } + /** * @abstract * @returns {Boolean} Whether the drawer implementation is supported by the browser. Must be overridden by extending classes. @@ -146,8 +194,7 @@ OpenSeadragon.DrawerBase = class DrawerBase{ */ minimumOverlapRequired() { return false; - } - + } /** * @abstract @@ -182,7 +229,7 @@ OpenSeadragon.DrawerBase = class DrawerBase{ * @private * */ - _checkForAPIOverrides(){ + _checkInterfaceImplementation(){ if(this._createDrawingElement === $.DrawerBase.prototype._createDrawingElement){ throw(new Error("[drawer]._createDrawingElement must be implemented by child class")); } diff --git a/src/htmldrawer.js b/src/htmldrawer.js index 824976ef..f2c5ec80 100644 --- a/src/htmldrawer.js +++ b/src/htmldrawer.js @@ -51,6 +51,8 @@ class HTMLDrawer extends OpenSeadragon.DrawerBase{ constructor(options){ super(options); + this.declareSupportedDataFormats("image"); + /** * The HTML element (div) that this drawer uses for drawing * @member {Element} canvas @@ -210,7 +212,7 @@ class HTMLDrawer extends OpenSeadragon.DrawerBase{ // content during animation of the container size. if ( !tile.element ) { - var image = tile.getImage(); + const image = this.getCompatibleData(tile); if (!image) { return; } diff --git a/src/tile.js b/src/tile.js index 071e5d53..b96a9687 100644 --- a/src/tile.js +++ b/src/tile.js @@ -510,7 +510,7 @@ $.Tile.prototype = { } if (!type) { - if (this.tiledImage && !this.tiledImage.__typeWarningReported) { + if (!this.tiledImage.__typeWarningReported) { $.console.warn(this, "[Tile.setCache] called without type specification. " + "Automated deduction is potentially unsafe: prefer specification of data type explicitly."); this.tiledImage.__typeWarningReported = true; @@ -520,10 +520,11 @@ $.Tile.prototype = { const writesToRenderingCache = key === this.cacheKey; if (writesToRenderingCache && _safely) { - //todo after-merge-aiosa decide dynamically - const conversion = $.convertor.getConversionPath(type, "context2d"); + // Need to get the supported type for rendering out of the active drawer. + const supportedTypes = this.tiledImage.viewer.drawer.getSupportedDataFormats(); + const conversion = $.convertor.getConversionPath(type, supportedTypes); $.console.assert(conversion, "[Tile.setCache] data was set for the default tile cache we are unable" + - "to render. Make sure OpenSeadragon.convertor was taught to convert type: " + type); + "to render. Make sure OpenSeadragon.convertor was taught to convert to (one of): " + type); } if (!this.__cutoff) { diff --git a/src/tilecache.js b/src/tilecache.js index ad0aa3fc..f20732d1 100644 --- a/src/tilecache.js +++ b/src/tilecache.js @@ -136,8 +136,9 @@ * @returns {OpenSeadragon.Promise} desired data type in promise, undefined if the cache was destroyed */ getDataAs(type = this._type, copy = true) { + const referenceTile = this._tiles[0]; if (this.loaded && type === this._type) { - return copy ? $.convertor.copy(this._data, type) : this._promise; + return copy ? $.convertor.copy(referenceTile, this._data, type) : this._promise; } return this._promise.then(data => { @@ -146,10 +147,10 @@ return undefined; } if (type !== this._type) { - return $.convertor.convert(data, this._type, type); + return $.convertor.convert(referenceTile, data, this._type, type); } if (copy) { //convert does not copy data if same type, do explicitly - return $.convertor.copy(data, type); + return $.convertor.copy(referenceTile, data, type); } return data; }); @@ -158,11 +159,15 @@ /** * Transform cache to desired type and get the data after conversion. * Does nothing if the type equals to the current type. Asynchronous. - * @param {string} type + * @param {string|[string]} type if array provided, the system will + * try to optimize for the best type to convert to. * @return {OpenSeadragon.Promise|*} */ transformTo(type = this._type) { - if (!this.loaded || type !== this._type) { + if (!this.loaded || + type !== this._type || + (Array.isArray(type) && !type.includes(this._type))) { + if (!this.loaded) { this._conversionJobQueue = this._conversionJobQueue || []; let resolver = null; @@ -173,7 +178,8 @@ if (this._destroyed) { return; } - if (type !== this._type) { + //must re-check types since we perform in a queue of conversion requests + if (type !== this._type || (Array.isArray(type) && !type.includes(this._type))) { //ensures queue gets executed after finish this._convert(this._type, type); this._promise.then(data => resolver(data)); @@ -351,10 +357,13 @@ /** * Private conversion that makes sure the cache knows its data is ready + * @param to array or a string - allowed types + * @param from string - type origin * @private */ _convert(from, to) { const convertor = $.convertor, + referenceTile = this._tiles[0], conversionPath = convertor.getConversionPath(from, to); if (!conversionPath) { $.console.error(`[OpenSeadragon.convertor.convert] Conversion conversion ${from} ---> ${to} cannot be done!`); @@ -372,7 +381,7 @@ return $.Promise.resolve(x); } let edge = conversionPath[i]; - return $.Promise.resolve(edge.transform(x)).then( + return $.Promise.resolve(edge.transform(referenceTile, x)).then( y => { if (!y) { $.console.error(`[OpenSeadragon.convertor.convert] data mid result falsey value (while converting using %s)`, edge); @@ -391,7 +400,8 @@ this.loaded = false; this._data = undefined; - this._type = to; + // Read target type from the conversion path: [edge.target] = Vertex, its value=type + this._type = conversionPath[stepCount - 1].target.value; this._promise = convert(originalData, 0); } }; @@ -657,6 +667,11 @@ this._tilesLoaded.splice( deleteAtIndex, 1 ); } + // Possible error: it can happen that unloaded tile gets to this stage. Should it even be allowed to happen? + if (!tile.loaded) { + return; + } + const tiledImage = tile.tiledImage; tile.unload(); diff --git a/src/tiledimage.js b/src/tiledimage.js index 26cf204b..20911365 100644 --- a/src/tiledimage.js +++ b/src/tiledimage.js @@ -2126,14 +2126,13 @@ $.extend($.TiledImage.prototype, $.EventSource.prototype, /** @lends OpenSeadrag ); //make sure cache data is ready for drawing, if not, request the desired format const cache = tile.getCache(tile.cacheKey), - // TODO: after-merge-aiosa dynamic type declaration from the drawer base class interface - requiredType = _this._drawer.useCanvas ? "context2d" : "image"; + requiredTypes = _this.viewer.drawer.getSupportedDataFormats(); if (!cache) { $.console.warn("Tile %s not cached at the end of tile-loaded event: tile will not be drawn - it has no data!", tile); resolver(tile); - } else if (cache.type !== requiredType) { + } else if (!requiredTypes.includes(cache.type)) { //initiate conversion as soon as possible if incompatible with the drawer - cache.transformTo(requiredType).then(_ => { + cache.transformTo(requiredTypes).then(_ => { tile.loading = false; tile.loaded = true; resolver(tile); diff --git a/src/viewer.js b/src/viewer.js index f1211315..2dccd285 100644 --- a/src/viewer.js +++ b/src/viewer.js @@ -379,7 +379,9 @@ $.Viewer = function( options ) { //update tiles item.opacity = 0; //prevent draw item.maxTilesPerFrame = 50; //todo based on image size and also number of images! - item._updateViewport(); + + //TODO check if the method is used correctly + item._updateLevelsForViewport(); item._needsDraw = true; //we did not draw item.opacity = origOpacity; item.maxTilesPerFrame = origMaxTiles; diff --git a/src/webgldrawer.js b/src/webgldrawer.js index bf40266d..620496dc 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -40,23 +40,23 @@ /** * @class OpenSeadragon.WebGLDrawer * @classdesc Default implementation of WebGLDrawer for an {@link OpenSeadragon.Viewer}. The WebGLDrawer - * loads tile data as textures to the graphics card as soon as it is available (via the tile-ready event), - * and unloads the data (via the image-unloaded event). The drawer utilizes a context-dependent two pass drawing pipeline. - * For the first pass, tile composition for a given TiledImage is always done using a canvas with a WebGL context. - * This allows tiles to be stitched together without seams or artifacts, without requiring a tile source with overlap. If overlap is present, - * overlapping pixels are discarded. The second pass copies all pixel data from the WebGL context onto an output canvas - * with a Context2d context. This allows applications to have access to pixel data and other functionality provided by - * Context2d, regardless of whether the CanvasDrawer or the WebGLDrawer is used. Certain options, including compositeOperation, - * clip, croppingPolygons, and debugMode are implemented using Context2d operations; in these scenarios, each TiledImage is - * drawn onto the output canvas immediately after the tile composition step (pass 1). Otherwise, for efficiency, all TiledImages - * are copied over to the output canvas at once, after all tiles have been composited for all images. + * defines its own data type that ensures textures are correctly loaded to and deleted from the GPU memory. + * The drawer utilizes a context-dependent two pass drawing pipeline. For the first pass, tile composition + * for a given TiledImage is always done using a canvas with a WebGL context. This allows tiles to be stitched + * together without seams or artifacts, without requiring a tile source with overlap. If overlap is present, + * overlapping pixels are discarded. The second pass copies all pixel data from the WebGL context onto an output + * canvas with a Context2d context. This allows applications to have access to pixel data and other functionality + * provided by Context2d, regardless of whether the CanvasDrawer or the WebGLDrawer is used. Certain options, + * including compositeOperation, clip, croppingPolygons, and debugMode are implemented using Context2d operations; + * in these scenarios, each TiledImage is drawn onto the output canvas immediately after the tile composition step + * (pass 1). Otherwise, for efficiency, all TiledImages are copied over to the output canvas at once, after all + * tiles have been composited for all images. * @param {Object} options - Options for this Drawer. * @param {OpenSeadragon.Viewer} options.viewer - The Viewer that owns this Drawer. * @param {OpenSeadragon.Viewport} options.viewport - Reference to Viewer viewport. * @param {Element} options.element - Parent element. * @param {Number} [options.debugGridColor] - See debugGridColor in {@link OpenSeadragon.Options} for details. */ - OpenSeadragon.WebGLDrawer = class WebGLDrawer extends OpenSeadragon.DrawerBase{ constructor(options){ super(options); @@ -76,24 +76,20 @@ // private members this._destroyed = false; - this._TextureMap = new Map(); - this._TileMap = new Map(); - this._gl = null; this._firstPass = null; this._secondPass = null; this._glFrameBuffer = null; this._renderToTexture = null; - this._glFramebufferToCanvasTransform = null; this._outputCanvas = null; this._outputContext = null; this._clippingCanvas = null; this._clippingContext = null; this._renderingCanvas = null; - // Add listeners for events that require modifying the scene or camera - this.viewer.addHandler("tile-ready", ev => this._tileReadyHandler(ev)); - this.viewer.addHandler("image-unloaded", ev => this._imageUnloadedHandler(ev)); + // Unique type per drawer: uploads texture to unique webgl context. + this._dataType = `${Date.now()}_TEX_2D`; + this._setupTextureHandlers(this._dataType); // Reject listening for the tile-drawing and tile-drawn events, which this drawer does not fire this.viewer.rejectEventHandler("tile-drawn", "The WebGLDrawer does not raise the tile-drawn event"); @@ -132,11 +128,6 @@ gl.bindRenderbuffer(gl.RENDERBUFFER, null); gl.bindFramebuffer(gl.FRAMEBUFFER, null); - let canvases = Array.from(this._TextureMap.keys()); - canvases.forEach(canvas => { - this._cleanupImageData(canvas); // deletes texture, removes from _TextureMap - }); - // Delete all our created resources gl.deleteBuffer(this._secondPass.bufferOutputPosition); gl.deleteFramebuffer(this._glFrameBuffer); @@ -316,15 +307,21 @@ let tile = tilesToDraw[tileIndex].tile; let indexInDrawArray = tileIndex % maxTextures; let numTilesToDraw = indexInDrawArray + 1; - let tileContext = tile.getCanvasContext(); - let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; - if(textureInfo){ - this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); - } else { - // console.log('No tile info', tile); + if ( !tile.loaded ) { + $.console.warn( + "Attempting to draw tile %s when it's not yet loaded.", + tile.toString() + ); + return; } - if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){ + const textureInfo = this.getCompatibleData(tile); + if (!textureInfo) { + return; + } + this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); + + if ((numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){ // We've filled up the buffers: time to draw this set of tiles // bind each tile's texture to the appropriate gl.TEXTURE# @@ -786,27 +783,12 @@ }); } - // private - _makeQuadVertexBuffer(left, right, top, bottom){ - return new Float32Array([ - left, bottom, - right, bottom, - left, top, - left, top, - right, bottom, - right, top]); - } + _setupTextureHandlers(thisType) { + const tex2DCompatibleLoader = (tile, data) => { + let tiledImage = tile.tiledImage; + //todo verify we are calling conversion just right amount of time! + // e.g. no upload of cpu-existing texture - // private - _tileReadyHandler(event){ - let tile = event.tile; - let tiledImage = event.tiledImage; - let tileContext = tile.getCanvasContext(); - let canvas = tileContext.canvas; - let textureInfo = this._TextureMap.get(canvas); - - // if this is a new image for us, create a texture - if(!textureInfo){ let gl = this._gl; // create a gl Texture for this tile and bind the canvas with the image data @@ -828,13 +810,6 @@ position = this._unitQuad; } - let textureInfo = { - texture: texture, - position: position, - }; - - // add it to our _TextureMap - this._TextureMap.set(canvas, textureInfo); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); // Set the parameters so we can render any size image. @@ -843,11 +818,55 @@ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - // Upload the image into the texture. - this._uploadImageData(tileContext); + try{ + // This depends on gl.TEXTURE_2D being bound to the texture + // associated with this canvas before calling this function + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, data); + } catch (e){ + $.console.error('Error uploading image data to WebGL', e); + } - } + // TextureInfo stored in the cache + return { + texture: texture, + position: position, + cpuData: data, + }; + }; + const tex2DCompatibleDestructor = textureInfo => { + if (textureInfo) { + this._gl.deleteTexture(textureInfo.texture); + } + }; + const dataRetrieval = (tile, data) => { + return data.cpuData; + }; + // Differentiate type also based on type used to upload data: we can support bidirectional conversion. + const c2dTexType = thisType + ":context2d", + imageTexType = thisType + ":image"; + + this.declareSupportedDataFormats(imageTexType, c2dTexType); + + // We should be OK uploading any of these types. + $.convertor.learn("context2d", c2dTexType, tex2DCompatibleLoader, 1, 2); + $.convertor.learn("image", imageTexType, tex2DCompatibleLoader, 1, 2); + $.convertor.learn(c2dTexType, "context2d", dataRetrieval, 1, 2); + $.convertor.learn(imageTexType, "image", dataRetrieval, 1, 2); + + $.convertor.learnDestroy(c2dTexType, tex2DCompatibleDestructor); + $.convertor.learnDestroy(imageTexType, tex2DCompatibleDestructor); + } + + // private + _makeQuadVertexBuffer(left, right, top, bottom){ + return new Float32Array([ + left, bottom, + right, bottom, + left, top, + left, top, + right, bottom, + right, top]); } // private @@ -865,43 +884,6 @@ }; } - // private - _uploadImageData(tileContext){ - - let gl = this._gl; - let canvas = tileContext.canvas; - - try{ - if(!canvas){ - throw('Tile context does not have a canvas', tileContext); - } - // This depends on gl.TEXTURE_2D being bound to the texture - // associated with this canvas before calling this function - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas); - } catch (e){ - $.console.error('Error uploading image data to WebGL', e); - } - } - - // private - _imageUnloadedHandler(event){ - let canvas = event.context2D.canvas; - this._cleanupImageData(canvas); - } - - // private - _cleanupImageData(tileCanvas){ - let textureInfo = this._TextureMap.get(tileCanvas); - //remove from the map - this._TextureMap.delete(tileCanvas); - - //release the texture from the GPU - if(textureInfo){ - this._gl.deleteTexture(textureInfo.texture); - } - - } - // private _setClip(rect){ this._clippingContext.beginPath(); @@ -1133,9 +1115,7 @@ return shaderProgram; } - }; - }( OpenSeadragon )); diff --git a/test/modules/basic.js b/test/modules/basic.js index 23810b5b..23b0378e 100644 --- a/test/modules/basic.js +++ b/test/modules/basic.js @@ -332,9 +332,10 @@ } ] } ); viewer.addOnceHandler('tiled-image-drawn', function(event) { - assert.ok(OpenSeadragon.isCanvasTainted(event.tiles[0].getCanvasContext().canvas), - "Canvas should be tainted."); - done(); + event.tiles[0].getCache().getDataAs("context2d", false).then(context => + assert.ok(OpenSeadragon.isCanvasTainted(context.canvas), + "Canvas should be tainted.") + ).then(done); }); } ); @@ -352,9 +353,10 @@ } ] } ); viewer.addOnceHandler('tiled-image-drawn', function(event) { - assert.ok(!OpenSeadragon.isCanvasTainted(event.tiles[0].getCanvasContext().canvas), - "Canvas should not be tainted."); - done(); + event.tiles[0].getCache().getDataAs("context2d", false).then(context => + assert.notOk(OpenSeadragon.isCanvasTainted(context.canvas), + "Canvas should be tainted.") + ).then(done); }); } ); @@ -376,9 +378,10 @@ crossOriginPolicy : false } ); viewer.addOnceHandler('tiled-image-drawn', function(event) { - assert.ok(OpenSeadragon.isCanvasTainted(event.tiles[0].getCanvasContext().canvas), - "Canvas should be tainted."); - done(); + event.tiles[0].getCache().getDataAs("context2d", false).then(context => + assert.ok(OpenSeadragon.isCanvasTainted(context.canvas), + "Canvas should be tainted.") + ).then(done); }); } ); @@ -400,9 +403,10 @@ } } ); viewer.addOnceHandler('tiled-image-drawn', function(event) { - assert.ok(!OpenSeadragon.isCanvasTainted(event.tiles[0].getCanvasContext().canvas), - "Canvas should not be tainted."); - done(); + event.tiles[0].getCache().getDataAs("context2d", false).then(context => + assert.notOk(OpenSeadragon.isCanvasTainted(context.canvas), + "Canvas should be tainted.") + ).then(done); }); } ); diff --git a/test/modules/tilecache.js b/test/modules/tilecache.js index 9eafaec9..35586beb 100644 --- a/test/modules/tilecache.js +++ b/test/modules/tilecache.js @@ -33,46 +33,46 @@ // other tests will interfere let typeAtoB = 0, typeBtoC = 0, typeCtoA = 0, typeDtoA = 0, typeCtoE = 0; //set all same costs to get easy testing, know which path will be taken - Convertor.learn(T_A, T_B, x => { + Convertor.learn(T_A, T_B, (tile, x) => { typeAtoB++; return x+1; }); - Convertor.learn(T_B, T_C, x => { + Convertor.learn(T_B, T_C, (tile, x) => { typeBtoC++; return x+1; }); - Convertor.learn(T_C, T_A, x => { + Convertor.learn(T_C, T_A, (tile, x) => { typeCtoA++; return x+1; }); - Convertor.learn(T_D, T_A, x => { + Convertor.learn(T_D, T_A, (tile, x) => { typeDtoA++; return x+1; }); - Convertor.learn(T_C, T_E, x => { + Convertor.learn(T_C, T_E, (tile, x) => { typeCtoE++; return x+1; }); //'Copy constructors' let copyA = 0, copyB = 0, copyC = 0, copyD = 0, copyE = 0; //also learn destructors - Convertor.learn(T_A, T_A,x => { + Convertor.learn(T_A, T_A,(tile, x) => { copyA++; return x+1; }); - Convertor.learn(T_B, T_B,x => { + Convertor.learn(T_B, T_B,(tile, x) => { copyB++; return x+1; }); - Convertor.learn(T_C, T_C,x => { + Convertor.learn(T_C, T_C,(tile, x) => { copyC++; return x-1; }); - Convertor.learn(T_D, T_D,x => { + Convertor.learn(T_D, T_D,(tile, x) => { copyD++; return x+1; }); - Convertor.learn(T_E, T_E,x => { + Convertor.learn(T_E, T_E,(tile, x) => { copyE++; return x+1; }); diff --git a/test/modules/type-conversion.js b/test/modules/type-conversion.js index 4f7616e5..42f5033a 100644 --- a/test/modules/type-conversion.js +++ b/test/modules/type-conversion.js @@ -31,23 +31,23 @@ let imageToCanvas = 0, srcToImage = 0, context2DtoImage = 0, canvasToContext2D = 0, imageToUrl = 0, canvasToUrl = 0; //set all same costs to get easy testing, know which path will be taken - Convertor.learn("__TEST__canvas", "__TEST__url", canvas => { + Convertor.learn("__TEST__canvas", "__TEST__url", (tile, canvas) => { canvasToUrl++; return canvas.toDataURL(); }, 1, 1); - Convertor.learn("__TEST__image", "__TEST__url", image => { + Convertor.learn("__TEST__image", "__TEST__url", (tile,image) => { imageToUrl++; return image.url; }, 1, 1); - Convertor.learn("__TEST__canvas", "__TEST__context2d", canvas => { + Convertor.learn("__TEST__canvas", "__TEST__context2d", (tile,canvas) => { canvasToContext2D++; return canvas.getContext("2d"); }, 1, 1); - Convertor.learn("__TEST__context2d", "__TEST__canvas", context2D => { + Convertor.learn("__TEST__context2d", "__TEST__canvas", (tile,context2D) => { context2DtoImage++; return context2D.canvas; }, 1, 1); - Convertor.learn("__TEST__image", "__TEST__canvas", image => { + Convertor.learn("__TEST__image", "__TEST__canvas", (tile,image) => { imageToCanvas++; const canvas = document.createElement( 'canvas' ); canvas.width = image.width; @@ -56,7 +56,7 @@ context.drawImage( image, 0, 0 ); return canvas; }, 1, 1); - Convertor.learn("__TEST__url", "__TEST__image", url => { + Convertor.learn("__TEST__url", "__TEST__image", (tile, url) => { return new Promise((resolve, reject) => { srcToImage++; const img = new Image(); @@ -68,7 +68,8 @@ let canvasDestroy = 0, imageDestroy = 0, contex2DDestroy = 0, urlDestroy = 0; //also learn destructors - Convertor.learnDestroy("__TEST__canvas", () => { + Convertor.learnDestroy("__TEST__canvas", canvas => { + canvas.width = canvas.height = 0; canvasDestroy++; }); Convertor.learnDestroy("__TEST__image", () => { @@ -145,20 +146,20 @@ context.drawImage( image, 0, 0 ); //copy URL - const URL2 = await Convertor.copy(URL, "url"); + const URL2 = await Convertor.copy(null, URL, "url"); //we cannot check if they are not the same object, strings are immutable (and we don't copy anyway :D ) test.equal(URL, URL2, "String copy is equal in data."); test.equal(typeof URL, typeof URL2, "Type of copies equals."); test.equal(URL.length, URL2.length, "Data length is also equal."); //copy context - const context2 = await Convertor.copy(context, "context2d"); + const context2 = await Convertor.copy(null, context, "context2d"); test.notEqual(context, context2, "Copy is not the same as original canvas."); test.equal(typeof context, typeof context2, "Type of copies equals."); test.equal(context.canvas.toDataURL(), context2.canvas.toDataURL(), "Data is equal."); //copy image - const image2 = await Convertor.copy(image, "image"); + const image2 = await Convertor.copy(null, image, "image"); test.notEqual(image, image2, "Copy is not the same as original image."); test.equal(typeof image, typeof image2, "Type of copies equals."); test.equal(image.src, image2.src, "Data is equal."); @@ -173,7 +174,7 @@ const done = test.async(); //load image object: url -> image - Convertor.convert("/test/data/A.png", "__TEST__url", "__TEST__image").then(i => { + Convertor.convert(null, "/test/data/A.png", "__TEST__url", "__TEST__image").then(i => { test.equal(OpenSeadragon.type(i), "image", "Got image object after conversion."); test.equal(srcToImage, 1, "Conversion happened."); @@ -182,14 +183,14 @@ test.equal(urlDestroy, 1, "Url destructor called."); test.equal(imageDestroy, 0, "Image destructor not called."); - return Convertor.convert(i, "__TEST__image", "__TEST__canvas"); + return Convertor.convert(null, i, "__TEST__image", "__TEST__canvas"); }).then(c => { //path image -> canvas test.equal(OpenSeadragon.type(c), "canvas", "Got canvas object after conversion."); test.equal(srcToImage, 1, "Conversion ulr->image did not happen."); test.equal(imageToCanvas, 1, "Conversion image->canvas happened."); test.equal(urlDestroy, 1, "Url destructor not called."); test.equal(imageDestroy, 0, "Image destructor not called unless we ask it."); - return Convertor.convert(c, "__TEST__canvas", "__TEST__image"); + return Convertor.convert(null, c, "__TEST__canvas", "__TEST__image"); }).then(i => { //path canvas, image: canvas -> url -> image test.equal(OpenSeadragon.type(i), "image", "Got image object after conversion."); test.equal(srcToImage, 2, "Conversion ulr->image happened."); @@ -314,7 +315,7 @@ const done = test.async(); let conversionHappened = false; - Convertor.learn("__TEST__url", "__TEST__longConversionProcessForTesting", value => { + Convertor.learn("__TEST__url", "__TEST__longConversionProcessForTesting", (tile, value) => { return new Promise((resolve, reject) => { setTimeout(() => { conversionHappened = true; @@ -358,7 +359,7 @@ const done = test.async(); let conversionHappened = false; - Convertor.learn("__TEST__url", "__TEST__longConversionProcessForTesting", value => { + Convertor.learn("__TEST__url", "__TEST__longConversionProcessForTesting", (tile, value) => { return new Promise((resolve, reject) => { setTimeout(() => { conversionHappened = true;