diff --git a/examples/custom-provider/server/CustomProvider.cjs b/examples/custom-provider/server/CustomProvider.cjs index 6e4292b14e..789ff82953 100644 --- a/examples/custom-provider/server/CustomProvider.cjs +++ b/examples/custom-provider/server/CustomProvider.cjs @@ -61,26 +61,14 @@ class MyCustomProvider { }, }) - if (!resp.ok) { - throw new Error(`Errornous HTTP response (${resp.status} ${resp.statusText})`) - } - return { stream: Readable.fromWeb(resp.body) } - } - - // eslint-disable-next-line class-methods-use-this - async size ({ id, token }) { - const resp = await fetch(`${BASE_URL}/photos/${id}`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }) + const contentLengthStr = resp.headers['content-length'] + const contentLength = parseInt(contentLengthStr, 10); + const size = !Number.isNaN(contentLength) && contentLength >= 0 ? contentLength : undefined; if (!resp.ok) { throw new Error(`Errornous HTTP response (${resp.status} ${resp.statusText})`) } - - const { size } = await resp.json() - return size + return { stream: Readable.fromWeb(resp.body), size } } } diff --git a/packages/@uppy/companion/src/server/controllers/googlePicker.js b/packages/@uppy/companion/src/server/controllers/googlePicker.js index b08d66a605..67f9462666 100644 --- a/packages/@uppy/companion/src/server/controllers/googlePicker.js +++ b/packages/@uppy/companion/src/server/controllers/googlePicker.js @@ -3,10 +3,9 @@ const assert = require('node:assert') const { startDownUpload } = require('../helpers/upload') const { validateURL } = require('../helpers/request') -const { getURLMeta } = require('../helpers/request') const logger = require('../logger') const { downloadURL } = require('../download') -const { getGoogleFileSize, streamGoogleFile } = require('../provider/google/drive'); +const { streamGoogleFile } = require('../provider/google/drive'); const { respondWithError } = require('../provider/error') @@ -26,14 +25,6 @@ const get = async (req, res) => { const { accessToken, platform, fileId } = req.body assert(platform === 'drive' || platform === 'photos'); - - const getSize = async () => { - if (platform === 'drive') { - return getGoogleFileSize({ id: fileId, token: accessToken }) - } - const { size } = await getURLMeta(req.body.url, allowLocalUrls, { headers: getAuthHeader(accessToken) }) - return size - } if (platform === 'photos' && !validateURL(req.body.url, allowLocalUrls)) { res.status(400).json({ error: 'Invalid URL' }) @@ -47,7 +38,7 @@ const get = async (req, res) => { return downloadURL(req.body.url, allowLocalUrls, req.id, { headers: getAuthHeader(accessToken) }) } - await startDownUpload({ req, res, getSize, download }) + await startDownUpload({ req, res, download, getSize: undefined }) } catch (err) { logger.error(err, 'controller.googlePicker.error', req.id) if (respondWithError(err, res)) return diff --git a/packages/@uppy/companion/src/server/controllers/url.js b/packages/@uppy/companion/src/server/controllers/url.js index 99b1e96667..5018f29186 100644 --- a/packages/@uppy/companion/src/server/controllers/url.js +++ b/packages/@uppy/companion/src/server/controllers/url.js @@ -54,15 +54,10 @@ const get = async (req, res) => { return } - async function getSize () { - const { size } = await getURLMeta(req.body.url, allowLocalUrls) - return size - } - const download = () => downloadURL(req.body.url, allowLocalUrls, req.id) try { - await startDownUpload({ req, res, getSize, download }) + await startDownUpload({ req, res, download, getSize: undefined }) } catch (err) { logger.error(err, 'controller.url.error', req.id) if (respondWithError(err, res)) return diff --git a/packages/@uppy/companion/src/server/helpers/upload.js b/packages/@uppy/companion/src/server/helpers/upload.js index b508c52dff..0f166070d3 100644 --- a/packages/@uppy/companion/src/server/helpers/upload.js +++ b/packages/@uppy/companion/src/server/helpers/upload.js @@ -6,12 +6,14 @@ async function startDownUpload({ req, res, getSize, download }) { const { stream, size: maybeSize } = await download() let size - // if the provider already knows the size, we can use that + // if we already know the size from the GET response content-length header, we can use that if (typeof maybeSize === 'number' && !Number.isNaN(maybeSize) && maybeSize > 0) { size = maybeSize } - // if not we need to get the size - if (size == null) { + // if not, we may need to explicitly get the size + // note that getSize might also return undefined/null, which is usually fine, it just means that + // the size is unknown and we cannot send the size to the Uploader + if (size == null && getSize != null) { size = await getSize() } const { clientSocketConnectTimeout } = req.companion.options diff --git a/packages/@uppy/companion/src/server/provider/Provider.js b/packages/@uppy/companion/src/server/provider/Provider.js index cfbcfbd973..f872ff563f 100644 --- a/packages/@uppy/companion/src/server/provider/Provider.js +++ b/packages/@uppy/companion/src/server/provider/Provider.js @@ -58,14 +58,16 @@ class Provider { } /** - * get the size of a certain file in the provider account + * first Companion will try to get the size from the content-length response header, + * if that fails, it will call this method to get the size. + * So if your provider has a different method for getting the size, you can return the size here * * @param {object} options * @returns {Promise} */ // eslint-disable-next-line class-methods-use-this,no-unused-vars async size (options) { - throw new Error('method not implemented') + return undefined } /** diff --git a/packages/@uppy/companion/src/server/provider/box/index.js b/packages/@uppy/companion/src/server/provider/box/index.js index 3488363082..e35d3a88d8 100644 --- a/packages/@uppy/companion/src/server/provider/box/index.js +++ b/packages/@uppy/companion/src/server/provider/box/index.js @@ -62,8 +62,8 @@ class Box extends Provider { return this.#withErrorHandling('provider.box.download.error', async () => { const stream = (await getClient({ token })).stream.get(`files/${id}/content`, { responseType: 'json' }) - await prepareStream(stream) - return { stream } + const { size } = await prepareStream(stream) + return { stream, size } }) } diff --git a/packages/@uppy/companion/src/server/provider/dropbox/index.js b/packages/@uppy/companion/src/server/provider/dropbox/index.js index d33e274ecf..1e0a2b4766 100644 --- a/packages/@uppy/companion/src/server/provider/dropbox/index.js +++ b/packages/@uppy/companion/src/server/provider/dropbox/index.js @@ -96,8 +96,8 @@ class DropBox extends Provider { responseType: 'json', }) - await prepareStream(stream) - return { stream } + const { size } = await prepareStream(stream) + return { stream, size } }) } diff --git a/packages/@uppy/companion/src/server/provider/facebook/index.js b/packages/@uppy/companion/src/server/provider/facebook/index.js index b31588c5b1..45cdfed4cb 100644 --- a/packages/@uppy/companion/src/server/provider/facebook/index.js +++ b/packages/@uppy/companion/src/server/provider/facebook/index.js @@ -1,7 +1,6 @@ const crypto = require('node:crypto'); const Provider = require('../Provider') -const { getURLMeta } = require('../../helpers/request') const logger = require('../../logger') const { adaptData, sortImages } = require('./adapter') const { withProviderErrorHandling } = require('../providerErrors') @@ -92,8 +91,8 @@ class Facebook extends Provider { return this.#withErrorHandling('provider.facebook.download.error', async () => { const url = await getMediaUrl({ secret: this.secret, token, id }) const stream = (await got).stream.get(url, { responseType: 'json' }) - await prepareStream(stream) - return { stream } + const { size } = await prepareStream(stream) + return { stream, size } }) } @@ -104,14 +103,6 @@ class Facebook extends Provider { throw new Error('call to thumbnail is not implemented') } - async size ({ id, token }) { - return this.#withErrorHandling('provider.facebook.size.error', async () => { - const url = await getMediaUrl({ secret: this.secret, token, id }) - const { size } = await getURLMeta(url) - return size - }) - } - async logout ({ token }) { return this.#withErrorHandling('provider.facebook.logout.error', async () => { await runRequestBatch({ diff --git a/packages/@uppy/companion/src/server/provider/google/drive/index.js b/packages/@uppy/companion/src/server/provider/google/drive/index.js index a4a790f3c5..d5ec1c57b3 100644 --- a/packages/@uppy/companion/src/server/provider/google/drive/index.js +++ b/packages/@uppy/companion/src/server/provider/google/drive/index.js @@ -74,20 +74,8 @@ async function streamGoogleFile({ token, id: idIn }) { stream = client.stream.get(`files/${encodeURIComponent(id)}`, { searchParams: { alt: 'media', supportsAllDrives: true }, responseType: 'json' }) } - await prepareStream(stream) - return { stream } -} - -async function getGoogleFileSize({ id, token }) { - const { mimeType, size } = await getStats({ id, token }) - - if (isGsuiteFile(mimeType)) { - // GSuite file sizes cannot be predetermined (but are max 10MB) - // e.g. Transfer-Encoding: chunked - return undefined - } - - return parseInt(size, 10) + const { size } = await prepareStream(stream) + return { stream, size } } /** @@ -185,13 +173,6 @@ class Drive extends Provider { return streamGoogleFile({ token, id }) }) } - - // eslint-disable-next-line class-methods-use-this - async size ({ id, token }) { - return withGoogleErrorHandling(Drive.oauthProvider, 'provider.drive.size.error', async () => ( - getGoogleFileSize({ id, token }) - )) - } } Drive.prototype.logout = logout @@ -200,5 +181,4 @@ Drive.prototype.refreshToken = refreshToken module.exports = { Drive, streamGoogleFile, - getGoogleFileSize, } diff --git a/packages/@uppy/companion/src/server/provider/instagram/graph/index.js b/packages/@uppy/companion/src/server/provider/instagram/graph/index.js index db124452df..13ab4048c4 100644 --- a/packages/@uppy/companion/src/server/provider/instagram/graph/index.js +++ b/packages/@uppy/companion/src/server/provider/instagram/graph/index.js @@ -1,5 +1,4 @@ const Provider = require('../../Provider') -const { getURLMeta } = require('../../../helpers/request') const logger = require('../../../logger') const adaptData = require('./adapter') const { withProviderErrorHandling } = require('../../providerErrors') @@ -55,8 +54,8 @@ class Instagram extends Provider { return this.#withErrorHandling('provider.instagram.download.error', async () => { const url = await getMediaUrl({ token, id }) const stream = (await got).stream.get(url, { responseType: 'json' }) - await prepareStream(stream) - return { stream } + const { size } = await prepareStream(stream) + return { stream, size } }) } @@ -67,14 +66,6 @@ class Instagram extends Provider { throw new Error('call to thumbnail is not implemented') } - async size ({ id, token }) { - return this.#withErrorHandling('provider.instagram.size.error', async () => { - const url = await getMediaUrl({ token, id }) - const { size } = await getURLMeta(url) - return size - }) - } - // eslint-disable-next-line class-methods-use-this async logout () { // access revoke is not supported by Instagram's API diff --git a/packages/@uppy/companion/src/server/provider/onedrive/index.js b/packages/@uppy/companion/src/server/provider/onedrive/index.js index 191222dd5f..2188c9bcbd 100644 --- a/packages/@uppy/companion/src/server/provider/onedrive/index.js +++ b/packages/@uppy/companion/src/server/provider/onedrive/index.js @@ -61,8 +61,8 @@ class OneDrive extends Provider { async download ({ id, token, query }) { return this.#withErrorHandling('provider.onedrive.download.error', async () => { const stream = (await getClient({ token })).stream.get(`${getRootPath(query)}/items/${id}/content`, { responseType: 'json' }) - await prepareStream(stream) - return { stream } + const { size } = await prepareStream(stream) + return { stream, size } }) } diff --git a/packages/@uppy/companion/src/server/provider/unsplash/index.js b/packages/@uppy/companion/src/server/provider/unsplash/index.js index b097e92287..d1c7cdaef2 100644 --- a/packages/@uppy/companion/src/server/provider/unsplash/index.js +++ b/packages/@uppy/companion/src/server/provider/unsplash/index.js @@ -1,5 +1,4 @@ const Provider = require('../Provider') -const { getURLMeta } = require('../../helpers/request') const adaptData = require('./adapter') const { withProviderErrorHandling } = require('../providerErrors') const { prepareStream } = require('../../helpers/utils') @@ -43,7 +42,7 @@ class Unsplash extends Provider { const { links: { download: url, download_location: attributionUrl } } = await getPhotoMeta(client, id) const stream = (await got).stream.get(url, { responseType: 'json' }) - await prepareStream(stream) + const { size } = await prepareStream(stream) // To attribute the author of the image, we call the `download_location` // endpoint to increment the download count on Unsplash. @@ -51,15 +50,7 @@ class Unsplash extends Provider { await client.get(attributionUrl, { prefixUrl: '', responseType: 'json' }) // finally, stream on! - return { stream } - }) - } - - async size ({ id, token }) { - return this.#withErrorHandling('provider.unsplash.size.error', async () => { - const { links: { download: url } } = await getPhotoMeta(await getClient({ token }), id) - const { size } = await getURLMeta(url) - return size + return { stream, size } }) } diff --git a/packages/@uppy/companion/src/server/provider/webdav/index.js b/packages/@uppy/companion/src/server/provider/webdav/index.js index f379aff592..b0f773754e 100644 --- a/packages/@uppy/companion/src/server/provider/webdav/index.js +++ b/packages/@uppy/companion/src/server/provider/webdav/index.js @@ -140,8 +140,10 @@ class WebdavProvider extends Provider { async download ({ id, providerUserSession }) { return this.withErrorHandling('provider.webdav.download.error', async () => { const client = await this.getClient({ providerUserSession }) + /** @type {any} */ + const stat = await client.stat(id) const stream = client.createReadStream(`/${id}`) - return { stream } + return { stream, size: stat.size } }) } @@ -152,17 +154,6 @@ class WebdavProvider extends Provider { throw new Error('call to thumbnail is not implemented') } - // eslint-disable-next-line - async size ({ id, token, providerUserSession }) { - return this.withErrorHandling('provider.webdav.size.error', async () => { - const client = await this.getClient({ providerUserSession }) - - /** @type {any} */ - const stat = await client.stat(id) - return stat.size - }) - } - // eslint-disable-next-line class-methods-use-this async withErrorHandling (tag, fn) { try { diff --git a/packages/@uppy/companion/src/server/provider/zoom/index.js b/packages/@uppy/companion/src/server/provider/zoom/index.js index b9f5ec84ce..200ce5e266 100644 --- a/packages/@uppy/companion/src/server/provider/zoom/index.js +++ b/packages/@uppy/companion/src/server/provider/zoom/index.js @@ -94,8 +94,8 @@ class Zoom extends Provider { if (!url) throw new Error('Download URL not found') const stream = client.stream.get(`${url}?access_token=${token}`, { prefixUrl: '', responseType: 'json' }) - await prepareStream(stream) - return { stream } + const { size } = await prepareStream(stream) + return { stream, size } }) }