diff --git a/README.md b/README.md index 2a24ac73..42b3d7f1 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,15 @@ main().catch((error) => { Save as `quickstart.js` and run `node quickstart.js`. [Create a free account](https://cloudinary.com/users/register_free) if you don't have one — or run `npx @cloudinary/cloud` to [provision one without signing up](docs/get-credentials.md). +`upload()` also accepts a `Buffer`, `Uint8Array`, `ArrayBuffer`, `Blob` or `File`. Pass `filename` +so that raw uploads keep their extension and `original_filename` is useful (the default is the +`File` name, or `"file"`): + +```js +const csv = Buffer.from('id,name\n1,sample\n'); +await cloudinary.uploader.upload(csv, { resource_type: 'raw', filename: 'report.csv' }); +``` + ## Common tasks - [Get Cloudinary credentials](docs/get-credentials.md) diff --git a/lib/uploader.js b/lib/uploader.js index eaa830ec..0739c312 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -3,7 +3,7 @@ const { extname, basename } = require('path'); -const Writable = require("stream").Writable; +const { Readable, Writable } = require("stream"); // eslint-disable-next-line import/order const { upload_prefix } = require("./config")(); @@ -52,12 +52,58 @@ exports.unsigned_upload = function unsigned_upload(file, upload_preset, callback }; exports.upload = function upload(file, callback, options = {}) { + // Read all options now. A later change to options must not change this upload. + if (isUploadData(file)) { + file = { + data: file instanceof ArrayBuffer ? Buffer.from(file) : Buffer.from(file.buffer, file.byteOffset, file.byteLength), + filename: options.filename || "file", + contentType: "application/octet-stream" + }; + } else if (isBlob(file)) { + file = { + blob: file, + filename: options.filename || file.name || "file", + contentType: file.type || "application/octet-stream" + }; + } return call_api("upload", callback, options, function () { let params = build_upload_params(options); return isRemoteUrl(file) ? [params, { file: file }] : [params, {}, file]; }); }; +function isUploadData(file) { + return file instanceof Uint8Array || file instanceof ArrayBuffer; +} + +// Find a Blob or File by its shape. Blobs from polyfills or other realms fail "instanceof Blob". +function isBlob(file) { + return isObject(file) && typeof file.arrayBuffer === "function" && typeof file.size === "number" && typeof file.type === "string"; +} + +function isUploadSource(file) { + return isObject(file) && (Buffer.isBuffer(file.data) || isBlob(file.blob)); +} + +// Send the Blob as a stream. If the runtime cannot do this, read the full Blob into memory. +function pipeBlob(blob, upload_stream, onError) { + let source = null; + try { + if (typeof blob.stream === "function" && typeof Readable.fromWeb === "function") { + source = Readable.fromWeb(blob.stream()); + } + } catch (error) { + source = null; + } + if (source) { + source.on('error', onError).pipe(upload_stream); + return; + } + Promise.resolve().then(() => blob.arrayBuffer()).then((arrayBuffer) => { + upload_stream.end(Buffer.from(arrayBuffer)); + }, onError); +} + exports.upload_large = function upload_large(path, callback, options = {}) { if ((path != null) && isRemoteUrl(path)) { // upload a remote file @@ -540,9 +586,8 @@ function post(url, post_data, boundary, file, callback, options) { let finish_buffer = Buffer.from("--" + boundary + "--", 'ascii'); let oauth_token = options.oauth_token || config().oauth_token; if ((file != null) || options.stream) { - // eslint-disable-next-line no-nested-ternary - let filename = options.stream ? options.filename ? options.filename : "file" : basename(file); - file_header = Buffer.from(encodeFilePart(boundary, 'application/octet-stream', 'file', filename), 'binary'); + let { filename, contentType } = getFileUploadOptions(file, options); + file_header = Buffer.from(encodeFilePart(boundary, contentType, 'file', filename), 'utf8'); } const parsedUrl = new URL(url); let post_options = { @@ -617,12 +662,19 @@ function post(url, post_data, boundary, file, callback, options) { } if (file != null) { post_request.write(file_header); - fs.createReadStream(file).on('error', function (error) { + let onReadError = function (error) { callback({ error: error }); return post_request.abort(); - }).pipe(upload_stream); + }; + if (!isUploadSource(file)) { + fs.createReadStream(file).on('error', onReadError).pipe(upload_stream); + } else if (file.blob) { + pipeBlob(file.blob, upload_stream, onReadError); + } else { + upload_stream.end(file.data); + } } else { post_request.write(finish_buffer); post_request.end(); @@ -630,6 +682,27 @@ function post(url, post_data, boundary, file, callback, options) { return true; } +function getFileUploadOptions(file, options = {}) { + if (options.stream) { + return { + filename: options.filename || "file", + contentType: "application/octet-stream" + }; + } + + if (isUploadSource(file)) { + return { + filename: file.filename, + contentType: file.contentType + }; + } + + return { + filename: basename(file), + contentType: "application/octet-stream" + }; +} + function encodeFieldPart(boundary, name, value) { return [ `--${boundary}\r\n`, @@ -640,10 +713,15 @@ function encodeFieldPart(boundary, name, value) { ].join(''); } +// Escape the filename as browsers do. A quote or a line break must not end the header. +function escapeFilename(filename) { + return String(filename).replace(/"/g, '%22').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + function encodeFilePart(boundary, type, name, filename) { return [ `--${boundary}\r\n`, - `Content-Disposition: form-data; name="${name}"; filename="${filename}"\r\n`, + `Content-Disposition: form-data; name="${name}"; filename="${escapeFilename(filename)}"\r\n`, `Content-Type: ${type}\r\n`, '\r\n', '' diff --git a/test/integration/api/uploader/uploader_spec.js b/test/integration/api/uploader/uploader_spec.js index ad98938f..82e4154f 100644 --- a/test/integration/api/uploader/uploader_spec.js +++ b/test/integration/api/uploader/uploader_spec.js @@ -50,6 +50,7 @@ const SAMPLE_IMAGE_URL_1 = "https://res.cloudinary.com/demo/image/upload/sample" const SAMPLE_IMAGE_URL_2 = "https://res.cloudinary.com/demo/image/upload/car" const { URL: NodeURL, URLSearchParams: NodeURLSearchParams } = require('url'); +const { Blob: NodeBlob, File: NodeFile } = require('buffer'); let cleanupJsdom; describe("uploader", function () { @@ -95,6 +96,102 @@ describe("uploader", function () { expect(result.signature).to.eql(expected_signature); }); }); + describe("in-memory uploads", function () { + it("should successfully upload a Buffer", function () { + const buffer = fs.readFileSync(IMAGE_FILE); + return cloudinary.v2.uploader.upload(buffer, { + tags: UPLOAD_TAGS + }).then(function (result) { + expect(result.width).to.eql(241); + expect(result.height).to.eql(51); + expect(result.format).to.eql("png"); + }); + }); + + it("should successfully upload a Uint8Array", function () { + const uint8Array = new Uint8Array(fs.readFileSync(IMAGE_FILE)); + return cloudinary.v2.uploader.upload(uint8Array, { + tags: UPLOAD_TAGS + }).then(function (result) { + expect(result.width).to.eql(241); + expect(result.height).to.eql(51); + expect(result.format).to.eql("png"); + }); + }); + + it("should successfully upload an ArrayBuffer", function () { + const buffer = fs.readFileSync(IMAGE_FILE); + const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); + return cloudinary.v2.uploader.upload(arrayBuffer, { + tags: UPLOAD_TAGS + }).then(function (result) { + expect(result.width).to.eql(241); + expect(result.height).to.eql(51); + expect(result.format).to.eql("png"); + }); + }); + + it("should successfully upload a Blob", function () { + if (typeof NodeBlob === "undefined") { + this.skip(); + } + const buffer = fs.readFileSync(IMAGE_FILE); + // jsdom-global replaces global.Blob. Use the Node Blob. + const blob = new NodeBlob([buffer], { type: "image/png" }); + return cloudinary.v2.uploader.upload(blob, { + tags: UPLOAD_TAGS + }).then(function (result) { + expect(result.width).to.eql(241); + expect(result.height).to.eql(51); + expect(result.format).to.eql("png"); + }); + }); + + it("should upload a raw buffer when resource_type is raw", function () { + const buffer = fs.readFileSync(RAW_FILE); + return cloudinary.v2.uploader.upload(buffer, { + resource_type: "raw", + filename: "report.docx", + use_filename: true, + tags: UPLOAD_TAGS + }).then(function (result) { + expect(result.resource_type).to.eql("raw"); + expect(result.original_filename).to.eql("report"); + expect(result.public_id).to.match(/^report_\w+\.docx$/); + }); + }); + + it("should keep a non-ASCII File name as original_filename", function () { + if (typeof NodeFile === "undefined") { + this.skip(); + } + const file = new NodeFile([fs.readFileSync(IMAGE_FILE)], "zdjęcie.png", { type: "image/png" }); + return cloudinary.v2.uploader.upload(file, { + use_filename: true, + tags: UPLOAD_TAGS + }).then(function (result) { + expect(result.original_filename).to.eql("zdjęcie"); + }); + }); + + it("should send buffer uploads without reading from the filesystem", function () { + const buffer = fs.readFileSync(IMAGE_FILE); + return helper.provideMockObjects(async function (mockXHR, writeSpy) { + const createReadStreamSpy = sinon.spy(fs, "createReadStream"); + try { + await cloudinary.v2.uploader.upload(buffer, { + filename: "buffer-upload.png", + tags: UPLOAD_TAGS + }).catch(helper.ignoreApiFailure); + sinon.assert.notCalled(createReadStreamSpy); + sinon.assert.calledWith(writeSpy, sinon.match((arg) => Buffer.isBuffer(arg) && arg.equals(buffer))); + sinon.assert.calledWith(writeSpy, sinon.match((arg) => Buffer.isBuffer(arg) && arg.toString("utf8").includes('filename="buffer-upload.png"'))); + } finally { + createReadStreamSpy.restore(); + } + }); + }); + }); it("should successfully upload with metadata", function () { return helper.provideMockObjects(async function (mockXHR, writeSpy, requestSpy) { await uploadImage({ metadata: METADATA_SAMPLE_DATA }).catch(helper.ignoreApiFailure); diff --git a/test/unit/upload_blob_spec.js b/test/unit/upload_blob_spec.js new file mode 100644 index 00000000..d2417ff0 --- /dev/null +++ b/test/unit/upload_blob_spec.js @@ -0,0 +1,166 @@ +const assert = require('assert'); +const sinon = require('sinon'); +const { Readable, Writable } = require('stream'); +const https = require('https'); + +const cloudinary = require('../../cloudinary'); +const createTestConfig = require('../testUtils/createTestConfig'); + +// Keep the bytes of the last request body. +let sentChunks = []; + +function sentBody() { + return Buffer.concat(sentChunks); +} + +// Give each request a fake HTTP 500 response. No network call occurs. +function fakeErrorRequest(options, onResponse) { + sentChunks = []; + const request = new Writable({ + write(chunk, encoding, done) { + sentChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding)); + done(); + } + }); + request.setTimeout = () => request; + request.abort = () => {}; + request.on('finish', () => { + const response = new Readable({ + read() {} + }); + response.statusCode = 500; + response.headers = {}; + onResponse(response); + response.push('{"error":{"message":"boom"}}'); + response.push(null); + }); + return request; +} + +// Node before 15.7 has no global Blob. Run the Blob-like tests there to test the arrayBuffer() fallback. +function skipWithoutBlob(test) { + if (typeof Blob === 'undefined') { + test.skip(); + } +} + +describe('upload of a Blob', function () { + let requestStub; + + beforeEach(function () { + cloudinary.config(createTestConfig()); + requestStub = sinon.stub(https, 'request').callsFake(fakeErrorRequest); + }); + + afterEach(function () { + requestStub.restore(); + }); + + it('should call the callback one time when the upload fails', async function () { + skipWithoutBlob(this); + const callback = sinon.spy(); + const blob = new Blob(['sample'], { type: 'text/plain' }); + + await assert.rejects(cloudinary.v2.uploader.upload(blob, callback), { message: 'boom' }); + + sinon.assert.calledOnce(callback); + sinon.assert.calledWith(callback, sinon.match({ message: 'boom', http_code: 500 })); + }); + + it('should use the options that were set when upload() was called', async function () { + skipWithoutBlob(this); + const options = { public_id: 'first' }; + const promise = cloudinary.v2.uploader.upload(new Blob(['sample']), options); + options.public_id = 'second'; + await promise.catch(() => {}); + + const body = sentBody().toString('utf8'); + assert.ok(body.includes('name="public_id"\r\n\r\nfirst\r\n')); + assert.ok(!body.includes('second')); + }); + + it('should upload a Blob-like object that has no stream()', async function () { + const blobLike = { + arrayBuffer: () => Promise.resolve(new Uint8Array(Buffer.from('shape-bytes')).buffer), + size: 11, + type: 'text/plain', + name: 'shape.txt' + }; + await cloudinary.v2.uploader.upload(blobLike).catch(() => {}); + + const body = sentBody().toString('utf8'); + assert.ok(body.includes('filename="shape.txt"\r\nContent-Type: text/plain\r\n\r\nshape-bytes\r\n--')); + }); + + it('should reject with { error } when the Blob cannot be read', async function () { + const callback = sinon.spy(); + const readError = new Error('read failed'); + const blobLike = { + arrayBuffer: () => Promise.reject(readError), + size: 1, + type: '' + }; + + // Node 9 has no assert.rejects. + const error = await cloudinary.v2.uploader.upload(blobLike, callback).then(() => null, (e) => e); + assert.strictEqual(error && error.error, readError); + sinon.assert.calledOnce(callback); + // The v2 callback gets the error itself, as for a file that cannot be read. + sinon.assert.calledWith(callback, readError); + }); + + it('should throw synchronously when cloud_name is missing', function () { + const blobLike = { + arrayBuffer: () => Promise.resolve(new ArrayBuffer(1)), + size: 1, + type: '' + }; + delete cloudinary.config().cloud_name; + try { + assert.throws(() => cloudinary.v2.uploader.upload(blobLike), /Must supply cloud_name/); + assert.throws(() => cloudinary.v2.uploader.upload(Buffer.from('sample')), /Must supply cloud_name/); + } finally { + cloudinary.config(true); + } + }); + + it('should escape the filename and send the header as UTF-8', async function () { + const names = { + 'a"b\r\nX-Injected: 1.png': 'a%22b%0D%0AX-Injected: 1.png', + 'zdjęcie.png': 'zdjęcie.png', + '日本.png': '日本.png' + }; + for (const [name, expected] of Object.entries(names)) { + // eslint-disable-next-line no-await-in-loop + await cloudinary.v2.uploader.upload(Buffer.from('x'), { filename: name }).catch(() => {}); + const body = sentBody(); + assert.ok(body.includes(Buffer.from(`filename="${expected}"\r\n`, 'utf8')), name); + assert.ok(!body.includes('\r\nX-Injected'), name); + } + }); + + it('should stream a native Blob and send all of its bytes', async function () { + if (typeof Blob === 'undefined' || typeof Blob.prototype.stream !== 'function' || typeof Readable.fromWeb !== 'function') { + this.skip(); + } + const bytes = require('crypto').randomBytes(256 * 1024); + const blob = new Blob([bytes], { type: 'application/pdf' }); + const arrayBufferSpy = sinon.spy(blob, 'arrayBuffer'); + await cloudinary.v2.uploader.upload(blob).catch(() => {}); + + sinon.assert.notCalled(arrayBufferSpy); + const body = sentBody(); + const header = Buffer.from('Content-Type: application/pdf\r\n\r\n'); + const start = body.indexOf(header) + header.length; + assert.ok(body.subarray(start, start + bytes.length).equals(bytes)); + assert.strictEqual(body.subarray(start + bytes.length, start + bytes.length + 4).toString(), '\r\n--'); + }); + + it('should send only the bytes of a Uint8Array view', async function () { + const source = Buffer.from('xxHELLOyy'); + const view = new Uint8Array(source.buffer, source.byteOffset + 2, 5); + await cloudinary.v2.uploader.upload(view).catch(() => {}); + + assert.ok(sentBody().toString('utf8').includes('\r\n\r\nHELLO\r\n--')); + }); +}); diff --git a/types/cloudinary_ts_spec.ts b/types/cloudinary_ts_spec.ts index 59826c74..54d4abca 100644 --- a/types/cloudinary_ts_spec.ts +++ b/types/cloudinary_ts_spec.ts @@ -900,6 +900,24 @@ cloudinary.v2.uploader.upload("ftp://user1:mypass@ftp.example.com/sample.jpg", console.log(result, error); }); +// $ExpectType Promise +cloudinary.v2.uploader.upload(Buffer.from("sample")); + +// $ExpectType Promise +cloudinary.v2.uploader.upload(new Uint8Array([1, 2, 3])); + +// $ExpectType Promise +cloudinary.v2.uploader.upload(new ArrayBuffer(8)); + +// $ExpectType Promise +cloudinary.v2.uploader.upload(new Blob(["sample"], {type: "text/plain"})); + +// $ExpectType Promise +cloudinary.v2.uploader.upload({arrayBuffer: async () => new ArrayBuffer(8), type: "text/csv", size: 8, name: "report.csv"}); + +// $ExpectType Promise +cloudinary.v2.uploader.upload(Buffer.from("a,b"), {resource_type: "raw", filename: "report.csv"}); + // $ExpectType UploadStream | Promise cloudinary.v2.uploader.upload_large("my_large_video.mp4", { diff --git a/types/index.d.ts b/types/index.d.ts index 90d417bc..14ad6412 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -516,6 +516,11 @@ declare module 'cloudinary' { eval?: string; exif?: boolean; faces?: boolean; + /** + * The filename to send for a Buffer, Uint8Array, ArrayBuffer or Blob upload, for example 'report.csv'. + * The default is the File name, or "file". Raw assets get their extension from this name. + */ + filename?: string; filename_override?: string; folder?: string; format?: VideoFormat | ImageFormat; @@ -1285,6 +1290,16 @@ declare module 'cloudinary' { /****************************** Upload API V2 Methods *************************************/ namespace uploader { + /** A Blob or File. The SDK finds it by its shape, so polyfill Blobs also work. */ + interface UploadBlob { + arrayBuffer(): Promise; + type: string; + size: number; + name?: string; + } + + type UploadFile = string | Buffer | Uint8Array | ArrayBuffer | UploadBlob; + function add_context(context: string, public_ids: string[], options?: { type?: DeliveryType, resource_type?: ResourceType @@ -1394,17 +1409,17 @@ declare module 'cloudinary' { function unsigned_image_upload_tag(field: string, upload_preset: string, options?: UploadApiOptions): Promise; - function unsigned_upload(file: string, upload_preset: string, options?: UploadApiOptions, callback?: ResponseCallback): Promise; + function unsigned_upload(file: UploadFile, upload_preset: string, options?: UploadApiOptions, callback?: ResponseCallback): Promise; - function unsigned_upload(file: string, upload_preset: string, callback?: ResponseCallback): Promise; + function unsigned_upload(file: UploadFile, upload_preset: string, callback?: ResponseCallback): Promise; function unsigned_upload_stream(upload_preset: string, options?: UploadApiOptions, callback?: ResponseCallback): UploadStream; function unsigned_upload_stream(upload_preset: string, callback?: ResponseCallback): UploadStream; - function upload(file: string, options?: UploadApiOptions, callback?: UploadResponseCallback): Promise; + function upload(file: UploadFile, options?: UploadApiOptions, callback?: UploadResponseCallback): Promise; - function upload(file: string, callback?: UploadResponseCallback): Promise; + function upload(file: UploadFile, callback?: UploadResponseCallback): Promise; function upload_chunked(path: string, options?: UploadApiOptions, callback?: UploadResponseCallback): UploadStream;