断点续传拷贝（Node.js SDK）--对象存储-火山引擎

文档中心

导航

断点续传拷贝（Node.js SDK）

最近更新时间：2024.02.26 17:05:19首次发布时间：2022.05.13 18:28:15

断点续传复制适用于通过 TOS Node.js SDK 在单个桶内或同区域的两个桶之间复制大对象的场景。TOS Node.js SDK 提供了断点续传拷贝的功能，借助本地 CheckPoint 的机制记录已成功复制的分段，当出现网络异常或机器故障等问题导致分段复制中断，可再次调用该接口以实现续传的效果。断点续传复制将待复制的对象分割为多个分段，并支持并发复制，待所有分段复制完成后，合并成完整的文件。您可以设置断点续传复制的分段大小、复制分段的线程数、事件回调函数等。同时也能在断点续传复制任务执行过程中，取消该任务。

注意事项

拷贝文件不支持跨区域的桶间拷贝。
拷贝对象时，账号必须具备源对象的读取权限和目标桶的写入权限。
拷贝对象时，可以保留所有元数据（默认值）或指定新的元数据。但 ACL 并未被保留，而是设置为私有。

示例代码

断点续传拷贝

以下代码用于断点续传拷贝 srcBucket 桶中 srcObject 对象到 dstBucket桶中，并设置对象对象名为 dstObject 以及失败后重入下载。若复制过程中返回网络超时的报错，则以相同参数调用 resumableCopyObject 后实现断点续传拷贝重入。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2（北京)为例，则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Name:', error.name);
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  try {
    const srcBucket = 'node-sdk-test-bucket'; // 源桶
    const srcObject = 'source-test'; // 源对象
    const dstBucket = 'node-sdk-copy-bucket'; // 目标桶
    const dstObject = 'copy-test'; // 目标对象

    await client.resumableCopyObject({
      bucket: dstBucket,
      key: dstObject,
      srcBucket,
      srcKey: srcObject,
      // 下载时指定分片大小 10M
      partSize: 10 * 1024 * 1024,
      // 分片下载任务并发数量
      taskNum: 5,
      // 指定断点续传临时文件路径
      checkpoint: './example_dir/example.checkpoint',
    });

    // 查询拷贝对象信息
    const headResult = await client.headObject({
      bucket: dstBucket,
      key: dstObject,
    });
    console.log('headResult', headResult);
  } catch (error) {
    handleError(error);
  }
}

main();

处理事件回调

以下代码用于自定义断点续传拷贝回调函数。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { ResumableCopyEventType, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2（北京)为例，则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Name:', error.name);
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  try {
    const srcBucket = 'node-sdk-test-bucket'; // 源桶
    const srcObject = 'source-test'; // 源对象
    const dstBucket = 'node-sdk-copy-bucket'; // 目标桶
    const dstObject = 'copy-test'; // 目标对象

    await client.resumableCopyObject({
      bucket: dstBucket,
      key: dstObject,
      srcBucket,
      srcKey: srcObject,
      // 下载时指定分片大小 10M
      partSize: 10 * 1024 * 1024,
      // 分片下载任务并发数量
      taskNum: 5,
      // 指定断点续传临时文件路径
      checkpoint: './example_dir/example.checkpoint',
      // 通过自定义方式设置回调函数查看拷贝进度
      copyEventListener: (event) => {
        switch (event.type) {
          case ResumableCopyEventType.CreateMultipartUploadSucceed: {
            console.log(`Resume Create Multipart Upload Success.`);
            break;
          }
          case ResumableCopyEventType.CreateMultipartUploadFailed: {
            console.log(`Resume Create Multipart Upload Fail.`);
            break;
          }
          case ResumableCopyEventType.UploadPartCopySucceed: {
            console.log(`Copy ${event.bucket} ${event.key} part success, CopyPartInfo:%o`, event.copyPartInfo);
            break;
          }
          case ResumableCopyEventType.UploadPartCopyFailed: {
            console.log(`Copy ${event.bucket} ${event.key} part fail, err:%o`, event.err);
            break;
          }
          case ResumableCopyEventType.UploadPartCopyAborted: {
            console.log(`Copy ${event.bucket} ${event.key} part aborted.`);
            break;
          }
          case ResumableCopyEventType.CompleteMultipartUploadSucceed: {
            console.log(`Copy ${event.bucket} ${event.key} success.`);
            break;
          }
          case ResumableCopyEventType.CompleteMultipartUploadFailed: {
            console.log(`Copy ${event.bucket} ${event.key} part fail, err: %o`, event.err);
            break;
          }
        }
      },
    });

    // 查询拷贝对象信息
    const headResult = await client.headObject({
      bucket: dstBucket,
      key: dstObject,
    });
    console.log('headResult', headResult);
  } catch (error) {
    handleError(error);
  }
}

main();

取消机制

以下代码用于暂停或取消断点续传拷贝任务。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { isCancel, CancelToken, ResumableCopyEventType, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';
import fsp from 'fs/promises';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2（北京)为例，则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Name:', error.name);
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  // checkpoint 文件路径
  const checkpointFilePath = './example_dir/example.checkpoint';

  try {
    const srcBucket = 'node-sdk-test-bucket'; // 源桶
    const srcObject = 'source-test'; // 源对象
    const dstBucket = 'node-sdk-copy-bucket'; // 目标桶
    const dstObject = 'copy-test'; // 目标对象

    try {
      // 生成 cancelTokenSource
      const cancelTokenSource = CancelToken.source();
      // 5 秒后取消任务
      setTimeout(() => {
        cancelTokenSource.cancel();
        console.log('cancel resumableCopyObject');
      }, 5_000);

      await client.resumableCopyObject({
        bucket: dstBucket,
        key: dstObject,
        srcBucket,
        srcKey: srcObject,
        // 下载时指定分片大小 10M
        partSize: 10 * 1024 * 1024,
        // 分片下载任务并发数量
        taskNum: 5,
        // 指定断点续传临时文件路径
        checkpoint: './example_dir/example.checkpoint',
        // 通过自定义方式设置回调函数查看拷贝进度
        copyEventListener: (event) => {
          switch (event.type) {
            case ResumableCopyEventType.CreateMultipartUploadSucceed: {
              console.log(`Resume Create Multipart Upload Success.`);
              break;
            }
            case ResumableCopyEventType.CreateMultipartUploadFailed: {
              console.log(`Resume Create Multipart Upload Fail.`);
              break;
            }
            case ResumableCopyEventType.UploadPartCopySucceed: {
              console.log(`Copy ${event.bucket} ${event.key} part success, CopyPartInfo:%o`, event.copyPartInfo);
              break;
            }
            case ResumableCopyEventType.UploadPartCopyFailed: {
              console.log(`Copy ${event.bucket} ${event.key} part fail, err:%o`, event.err);
              break;
            }
            case ResumableCopyEventType.UploadPartCopyAborted: {
              console.log(`Copy ${event.bucket} ${event.key} part aborted.`);
              break;
            }
            case ResumableCopyEventType.CompleteMultipartUploadSucceed: {
              console.log(`Copy ${event.bucket} ${event.key} success.`);
              break;
            }
            case ResumableCopyEventType.CompleteMultipartUploadFailed: {
              console.log(`Copy ${event.bucket} ${event.key} part fail, err: %o`, event.err);
              break;
            }
          }
        },
        cancelToken: cancelTokenSource.token,
      });
    } catch (error) {
      if (!isCancel(error)) {
        throw error;
      }

      const checkpointFileContent = await fsp.readFile(checkpointFilePath, 'utf-8');
      console.log(`checkpoint file's content after cancel: `, checkpointFileContent);

      console.log('Continue to copy');
      // 可以通过 checkpoint File 断点续传拷贝
      await client.resumableCopyObject({
        bucket: dstBucket,
        key: dstObject,
        srcBucket,
        srcKey: srcObject,
        // 下载时指定分片大小 10M
        partSize: 10 * 1024 * 1024,
        // 分片下载任务并发数量
        taskNum: 5,
        // 指定断点续传临时文件路径
        checkpoint: './example_dir/example.checkpoint',
        // 通过自定义方式设置回调函数查看拷贝进度
        copyEventListener: (event) => {
          switch (event.type) {
            case ResumableCopyEventType.CreateMultipartUploadSucceed: {
              console.log(`Resume Create Multipart Upload Success.`);
              break;
            }
            case ResumableCopyEventType.CreateMultipartUploadFailed: {
              console.log(`Resume Create Multipart Upload Fail.`);
              break;
            }
            case ResumableCopyEventType.UploadPartCopySucceed: {
              console.log(`Copy ${event.bucket} ${event.key} part success, CopyPartInfo:%o`, event.copyPartInfo);
              break;
            }
            case ResumableCopyEventType.UploadPartCopyFailed: {
              console.log(`Copy ${event.bucket} ${event.key} part fail, err:%o`, event.err);
              break;
            }
            case ResumableCopyEventType.UploadPartCopyAborted: {
              console.log(`Copy ${event.bucket} ${event.key} part aborted.`);
              break;
            }
            case ResumableCopyEventType.CompleteMultipartUploadSucceed: {
              console.log(`Copy ${event.bucket} ${event.key} success.`);
              break;
            }
            case ResumableCopyEventType.CompleteMultipartUploadFailed: {
              console.log(`Copy ${event.bucket} ${event.key} part fail, err: %o`, event.err);
              break;
            }
          }
        },
      });

      // 查询拷贝对象信息
      const headResult = await client.headObject({
        bucket: dstBucket,
        key: dstObject,
      });
      console.log('headResult', headResult);
    }
  } catch (error) {
    handleError(error);
  }
}

main();