object.js 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. const debug = require('debug')('ali-oss:object');
  2. const fs = require('fs');
  3. const is = require('is-type-of');
  4. const copy = require('copy-to');
  5. const path = require('path');
  6. const mime = require('mime');
  7. const callback = require('./common/callback');
  8. const { Transform } = require('stream');
  9. const pump = require('pump');
  10. const { isBuffer } = require('./common/utils/isBuffer');
  11. const { retry } = require('./common/utils/retry');
  12. const { obj2xml } = require('./common/utils/obj2xml');
  13. const proto = exports;
  14. /**
  15. * Object operations
  16. */
  17. /**
  18. * append an object from String(file path)/Buffer/ReadableStream
  19. * @param {String} name the object key
  20. * @param {Mixed} file String(file path)/Buffer/ReadableStream
  21. * @param {Object} options
  22. * @return {Object}
  23. */
  24. proto.append = async function append(name, file, options) {
  25. options = options || {};
  26. if (options.position === undefined) options.position = '0';
  27. options.subres = {
  28. append: '',
  29. position: options.position
  30. };
  31. options.method = 'POST';
  32. const result = await this.put(name, file, options);
  33. result.nextAppendPosition = result.res.headers['x-oss-next-append-position'];
  34. return result;
  35. };
  36. /**
  37. * put an object from String(file path)/Buffer/ReadableStream
  38. * @param {String} name the object key
  39. * @param {Mixed} file String(file path)/Buffer/ReadableStream
  40. * @param {Object} options
  41. * {Object} options.callback The callback parameter is composed of a JSON string encoded in Base64
  42. * {String} options.callback.url the OSS sends a callback request to this URL
  43. * {String} options.callback.host The host header value for initiating callback requests
  44. * {String} options.callback.body The value of the request body when a callback is initiated
  45. * {String} options.callback.contentType The Content-Type of the callback requests initiatiated
  46. * {Object} options.callback.customValue Custom parameters are a map of key-values, e.g:
  47. * customValue = {
  48. * key1: 'value1',
  49. * key2: 'value2'
  50. * }
  51. * @return {Object}
  52. */
  53. proto.put = async function put(name, file, options) {
  54. let content;
  55. options = options || {};
  56. name = this._objectName(name);
  57. if (isBuffer(file)) {
  58. content = file;
  59. } else if (is.string(file)) {
  60. const stats = fs.statSync(file);
  61. if (!stats.isFile()) {
  62. throw new Error(`${file} is not file`);
  63. }
  64. options.mime = options.mime || mime.getType(path.extname(file));
  65. options.contentLength = await this._getFileSize(file);
  66. const getStream = () => fs.createReadStream(file);
  67. const putStreamStb = (objectName, makeStream, configOption) => {
  68. return this.putStream(objectName, makeStream(), configOption);
  69. };
  70. return await retry(putStreamStb, this.options.retryMax, {
  71. errorHandler: err => {
  72. const _errHandle = _err => {
  73. const statusErr = [-1, -2].includes(_err.status);
  74. const requestErrorRetryHandle = this.options.requestErrorRetryHandle || (() => true);
  75. return statusErr && requestErrorRetryHandle(_err);
  76. };
  77. if (_errHandle(err)) return true;
  78. return false;
  79. }
  80. })(name, getStream, options);
  81. } else if (is.readableStream(file)) {
  82. return await this.putStream(name, file, options);
  83. } else {
  84. throw new TypeError('Must provide String/Buffer/ReadableStream for put.');
  85. }
  86. options.headers = options.headers || {};
  87. this._convertMetaToHeaders(options.meta, options.headers);
  88. const method = options.method || 'PUT';
  89. const params = this._objectRequestParams(method, name, options);
  90. callback.encodeCallback(params, options);
  91. params.mime = options.mime;
  92. params.content = content;
  93. params.successStatuses = [200];
  94. const result = await this.request(params);
  95. const ret = {
  96. name,
  97. url: this._objectUrl(name),
  98. res: result.res
  99. };
  100. if (params.headers && params.headers['x-oss-callback']) {
  101. ret.data = JSON.parse(result.data.toString());
  102. }
  103. return ret;
  104. };
  105. /**
  106. * put an object from ReadableStream. If `options.contentLength` is
  107. * not provided, chunked encoding is used.
  108. * @param {String} name the object key
  109. * @param {Readable} stream the ReadableStream
  110. * @param {Object} options
  111. * @return {Object}
  112. */
  113. proto.putStream = async function putStream(name, stream, options) {
  114. options = options || {};
  115. options.headers = options.headers || {};
  116. name = this._objectName(name);
  117. if (options.contentLength) {
  118. options.headers['Content-Length'] = options.contentLength;
  119. } else {
  120. options.headers['Transfer-Encoding'] = 'chunked';
  121. }
  122. this._convertMetaToHeaders(options.meta, options.headers);
  123. const method = options.method || 'PUT';
  124. const params = this._objectRequestParams(method, name, options);
  125. callback.encodeCallback(params, options);
  126. params.mime = options.mime;
  127. const transform = new Transform();
  128. // must remove http stream header for signature
  129. transform._transform = function _transform(chunk, encoding, done) {
  130. this.push(chunk);
  131. done();
  132. };
  133. params.stream = pump(stream, transform);
  134. params.successStatuses = [200];
  135. const result = await this.request(params);
  136. const ret = {
  137. name,
  138. url: this._objectUrl(name),
  139. res: result.res
  140. };
  141. if (params.headers && params.headers['x-oss-callback']) {
  142. ret.data = JSON.parse(result.data.toString());
  143. }
  144. return ret;
  145. };
  146. proto.getStream = async function getStream(name, options) {
  147. options = options || {};
  148. if (options.process) {
  149. options.subres = options.subres || {};
  150. options.subres['x-oss-process'] = options.process;
  151. }
  152. const params = this._objectRequestParams('GET', name, options);
  153. params.customResponse = true;
  154. params.successStatuses = [200, 206, 304];
  155. const result = await this.request(params);
  156. return {
  157. stream: result.res,
  158. res: {
  159. status: result.status,
  160. headers: result.headers
  161. }
  162. };
  163. };
  164. proto.putMeta = async function putMeta(name, meta, options) {
  165. return await this.copy(name, name, {
  166. meta: meta || {},
  167. timeout: options && options.timeout,
  168. ctx: options && options.ctx
  169. });
  170. };
  171. proto.list = async function list(query, options) {
  172. // prefix, marker, max-keys, delimiter
  173. const params = this._objectRequestParams('GET', '', options);
  174. params.query = query;
  175. params.xmlResponse = true;
  176. params.successStatuses = [200];
  177. const result = await this.request(params);
  178. let objects = result.data.Contents || [];
  179. const that = this;
  180. if (objects) {
  181. if (!Array.isArray(objects)) {
  182. objects = [objects];
  183. }
  184. objects = objects.map(obj => ({
  185. name: obj.Key,
  186. url: that._objectUrl(obj.Key),
  187. lastModified: obj.LastModified,
  188. etag: obj.ETag,
  189. type: obj.Type,
  190. size: Number(obj.Size),
  191. storageClass: obj.StorageClass,
  192. owner: {
  193. id: obj.Owner.ID,
  194. displayName: obj.Owner.DisplayName
  195. }
  196. }));
  197. }
  198. let prefixes = result.data.CommonPrefixes || null;
  199. if (prefixes) {
  200. if (!Array.isArray(prefixes)) {
  201. prefixes = [prefixes];
  202. }
  203. prefixes = prefixes.map(item => item.Prefix);
  204. }
  205. return {
  206. res: result.res,
  207. objects,
  208. prefixes,
  209. nextMarker: result.data.NextMarker || null,
  210. isTruncated: result.data.IsTruncated === 'true'
  211. };
  212. };
  213. proto.listV2 = async function listV2(query = {}, options = {}) {
  214. const continuation_token = query['continuation-token'] || query.continuationToken;
  215. delete query['continuation-token'];
  216. delete query.continuationToken;
  217. if (continuation_token) {
  218. options.subres = Object.assign(
  219. {
  220. 'continuation-token': continuation_token
  221. },
  222. options.subres
  223. );
  224. }
  225. const params = this._objectRequestParams('GET', '', options);
  226. params.query = Object.assign({ 'list-type': 2 }, query);
  227. delete params.query['continuation-token'];
  228. delete query.continuationToken;
  229. params.xmlResponse = true;
  230. params.successStatuses = [200];
  231. const result = await this.request(params);
  232. let objects = result.data.Contents || [];
  233. const that = this;
  234. if (objects) {
  235. if (!Array.isArray(objects)) {
  236. objects = [objects];
  237. }
  238. objects = objects.map(obj => {
  239. let owner = null;
  240. if (obj.Owner) {
  241. owner = {
  242. id: obj.Owner.ID,
  243. displayName: obj.Owner.DisplayName
  244. };
  245. }
  246. return {
  247. name: obj.Key,
  248. url: that._objectUrl(obj.Key),
  249. lastModified: obj.LastModified,
  250. etag: obj.ETag,
  251. type: obj.Type,
  252. size: Number(obj.Size),
  253. storageClass: obj.StorageClass,
  254. owner
  255. };
  256. });
  257. }
  258. let prefixes = result.data.CommonPrefixes || null;
  259. if (prefixes) {
  260. if (!Array.isArray(prefixes)) {
  261. prefixes = [prefixes];
  262. }
  263. prefixes = prefixes.map(item => item.Prefix);
  264. }
  265. return {
  266. res: result.res,
  267. objects,
  268. prefixes,
  269. isTruncated: result.data.IsTruncated === 'true',
  270. keyCount: +result.data.KeyCount,
  271. continuationToken: result.data.ContinuationToken || null,
  272. nextContinuationToken: result.data.NextContinuationToken || null
  273. };
  274. };
  275. /**
  276. * Restore Object
  277. * @param {String} name the object key
  278. * @param {Object} options {type : Archive or ColdArchive}
  279. * @returns {{res}}
  280. */
  281. proto.restore = async function restore(name, options = { type: 'Archive' }) {
  282. options = options || {};
  283. options.subres = Object.assign({ restore: '' }, options.subres);
  284. if (options.versionId) {
  285. options.subres.versionId = options.versionId;
  286. }
  287. const params = this._objectRequestParams('POST', name, options);
  288. if (options.type === 'ColdArchive') {
  289. const paramsXMLObj = {
  290. RestoreRequest: {
  291. Days: options.Days ? options.Days : 2,
  292. JobParameters: {
  293. Tier: options.JobParameters ? options.JobParameters : 'Standard'
  294. }
  295. }
  296. };
  297. params.content = obj2xml(paramsXMLObj, {
  298. headers: true
  299. });
  300. params.mime = 'xml';
  301. }
  302. params.successStatuses = [202];
  303. const result = await this.request(params);
  304. return {
  305. res: result.res
  306. };
  307. };
  308. proto._objectUrl = function _objectUrl(name) {
  309. return this._getReqUrl({ bucket: this.options.bucket, object: name });
  310. };
  311. /**
  312. * generator request params
  313. * @return {Object} params
  314. *
  315. * @api private
  316. */
  317. proto._objectRequestParams = function (method, name, options) {
  318. if (!this.options.bucket && !this.options.cname) {
  319. throw new Error('Please create a bucket first');
  320. }
  321. options = options || {};
  322. name = this._objectName(name);
  323. const params = {
  324. object: name,
  325. bucket: this.options.bucket,
  326. method,
  327. subres: options && options.subres,
  328. timeout: options && options.timeout,
  329. ctx: options && options.ctx
  330. };
  331. if (options.headers) {
  332. params.headers = {};
  333. copy(options.headers).to(params.headers);
  334. }
  335. return params;
  336. };
  337. proto._objectName = function (name) {
  338. return name.replace(/^\/+/, '');
  339. };
  340. proto._statFile = function (filepath) {
  341. return new Promise((resolve, reject) => {
  342. fs.stat(filepath, (err, stats) => {
  343. if (err) {
  344. reject(err);
  345. } else {
  346. resolve(stats);
  347. }
  348. });
  349. });
  350. };
  351. proto._convertMetaToHeaders = function (meta, headers) {
  352. if (!meta) {
  353. return;
  354. }
  355. Object.keys(meta).forEach(k => {
  356. headers[`x-oss-meta-${k}`] = meta[k];
  357. });
  358. };
  359. proto._deleteFileSafe = function (filepath) {
  360. return new Promise(resolve => {
  361. fs.exists(filepath, exists => {
  362. if (!exists) {
  363. resolve();
  364. } else {
  365. fs.unlink(filepath, err => {
  366. if (err) {
  367. debug('unlink %j error: %s', filepath, err);
  368. }
  369. resolve();
  370. });
  371. }
  372. });
  373. });
  374. };