分片上传（.NET SDK）--对象存储-火山引擎

文档中心

立即注册

导航

分片上传（.NET SDK）

最近更新时间：2024.02.04 18:31:07首次发布时间：2023.08.24 14:41:47

如果需要上传较大的对象，建议分成多个数据块（part）来分别上传，最后调用合并分片将上传的数据块合并为一个对象。

注意事项

分片上传前，您需要确保账号拥有 tos:PutObject 权限，具体操作，请参见权限配置指南。
取消分片上传任务时，您必须具备 tos:AbortMultipartUpload 权限，具体操作,请参见权限配置指南。
上传对象时，对象名必须满足命名规范，详细信息，请参见对象的命名规范。
TOS 是面向海量存储设计的分布式对象存储产品，内部分区存储了对象索引数据，为横向扩展您上传对象和下载对象时的最大吞吐量，和减小热点分区的概率，请您避免使用字典序递增的对象命名方式，详细信息,请参见性能优化。
如果桶中已经存在同名对象，则新对象会覆盖已有的对象。桶开启多版本的场景下，则会保留原有对象，生成一个新版本号用于标识新上传的对象。

分片上传步骤

使用 TOS .NET SDK 进行分片上传包含以下三个步骤。

初始化分片上传任务
调用 createMultipartUpload 接口返回 TOS 创建的全局唯一的uploadId。
上传分片
调用 uploadPart 接口上传分片数据。
说明
- 对于同一个分片上传任务（通过 uploadId 标识），分片编号（partNumber）标识了该分片在整个对象中的相对位置。若通过同一分片编号多次上传数据，TOS 会覆盖已有的数据，并以最后一次上传的数据为准。
- uploadPart 接口返回分片数据的 MD5 值，可通过 ETag 字段获取。合并分片时，您需指定当前分片上传任务中已上传的所有分片信息（分片编号、ETag值）。
完成分片上传
所有分片上传完成后，调用 completeMultipartUpload 接口将所有分片合并成一个完整的对象。

示例代码

分片上传完整过程

以下代码用于将本地文件通过分片的方式上传完整过程，并在上传时指定 ACL 为 Private、存储类型为低频存储以及添加自定义元数据。

using System;
using System.Collections.Generic;
using System.IO;
using TOS;
using TOS.Error;
using TOS.Model;

namespace ConsoleApp
{
    internal class Program
    {
        private static void Main(string[] args)
        {
            var ak = Environment.GetEnvironmentVariable("TOS_ACCESS_KEY");
            var sk = Environment.GetEnvironmentVariable("TOS_SECRET_KEY");
            // endpoint 若没有指定HTTP协议（HTTP/HTTPS），默认使用 HTTPS
            // Bucket 的 Endpoint，以华北2（北京）为例：https://tos-cn-beijing.volces.com
            var endpoint = "https://tos-cn-beijing.volces.com";
            var region = "cn-beijing";
            // 填写 BucketName
            var bucketName = "*** Provide your bucket name ***";
            // 将文件上传到 example_dir 目录下的 example.txt 文件
            var objectKey = "example_dir/example.txt";
            // 需要上传的本地文件
            var localFileName = "/usr/local/test.txt";

            // 创建TOSClient实例
            var client = TosClientBuilder.Builder().SetAk(ak).SetSk(sk).SetEndpoint(endpoint).SetRegion(region).Build();

            // 初始化分片，指定对象权限为私有，存储类型为低频并设置元数据信息
            var uploadID = "";
            try
            {
                var createMultipartUploadInput = new CreateMultipartUploadInput
                {
                    Bucket = bucketName,
                    Key = objectKey,
                    ACL = ACLType.ACLPrivate,
                    StorageClass = StorageClassType.StorageClassIa,
                    Meta = new Dictionary<string, string> { { "key", "value" } }
                };
                var createMultipartUploadOutput = client.CreateMultipartUpload(createMultipartUploadInput);
                uploadID = createMultipartUploadOutput.UploadID;
                Console.WriteLine("CreateMultipartUpload succeeded, request id {0}",
                    createMultipartUploadOutput.RequestID);
                Console.WriteLine("CreateMultipartUpload succeeded, upload id {0}", uploadID);
            }
            catch (TosServerException ex)
            {
                Console.WriteLine("CreateMultipartUpload failed, request id {0}", ex.RequestID);
                Console.WriteLine("CreateMultipartUpload failed, status code {0}", ex.StatusCode);
                Console.WriteLine("CreateMultipartUpload failed, response error code {0}", ex.Code);
                Console.WriteLine("CreateMultipartUpload failed, response error message {0}", ex.Message);
            }
            catch (TosClientException ex)
            {
                Console.WriteLine("CreateMultipartUpload failed, error message {0}", ex.Message);
            }
            catch (Exception ex)
            {
                Console.WriteLine("CreateMultipartUpload failed, {0}", ex.Message);
            }

            // part size 大小设置为 20M
            long partSize = 20 * 1024 * 1024;
            // 计算分片信息
            var fi = new FileInfo(localFileName);
            var fileSize = fi.Length;
            var partCount = fileSize / partSize;
            if (fileSize % partSize != 0) partCount++;
            var parts = new UploadedPart[partCount];

            // 开始分片上传
            try
            {
                using (var fileStream = File.Open(localFileName, FileMode.Open, FileAccess.Read))
                {
                    for (var i = 0; i < partCount; i++)
                    {
                        var offset = partSize * i;
                        // 定位到本次上传的起始位置。
                        fileStream.Seek(offset, 0);
                        // 计算本次上传的分片大小，最后一片为剩余的数据大小。
                        var contentSize = partSize < fileSize - offset ? partSize : fileSize - offset;
                        var UploadPartInput = new UploadPartInput
                        {
                            Bucket = bucketName,
                            Key = objectKey,
                            UploadID = uploadID,
                            // partNumber 编号从 1 开始
                            PartNumber = i + 1,
                            Content = fileStream,
                            ContentLength = contentSize
                        };
                        var uploadPartOutput = client.UploadPart(UploadPartInput);
                        parts[i] = new UploadedPart { PartNumber = i + 1, ETag = uploadPartOutput.ETag };
                        Console.WriteLine("UploadPart succeeded, request id {0}", uploadPartOutput.RequestID);
                    }
                }
            }
            catch (TosServerException ex)
            {
                Console.WriteLine("UploadPart failed, request id {0}", ex.RequestID);
                Console.WriteLine("UploadPart failed, status code {0}", ex.StatusCode);
                Console.WriteLine("UploadPart failed, response error code {0}", ex.Code);
                Console.WriteLine("UploadPart failed, response error message {0}", ex.Message);
            }
            catch (TosClientException ex)
            {
                Console.WriteLine("UploadPart failed, error message {0}", ex.Message);
            }
            catch (Exception ex)
            {
                Console.WriteLine("UploadPart failed, {0}", ex.Message);
            }

            // 合并分片
            try
            {
                var completeMultipartUploadInput = new CompleteMultipartUploadInput
                {
                    Bucket = bucketName,
                    Key = objectKey,
                    UploadID = uploadID,
                    Parts = parts
                };
                var completeMultipartUploadOutput = client.CompleteMultipartUpload(completeMultipartUploadInput);
                Console.WriteLine("CompleteMultipartUpload succeeded, request id {0}",
                    completeMultipartUploadOutput.RequestID);
            }
            catch (TosServerException ex)
            {
                Console.WriteLine("CompleteMultipartUpload failed, request id {0}", ex.RequestID);
                Console.WriteLine("CompleteMultipartUpload failed, status code {0}", ex.StatusCode);
                Console.WriteLine("CompleteMultipartUpload failed, response error code {0}", ex.Code);
                Console.WriteLine("CompleteMultipartUpload failed, response error message {0}", ex.Message);
            }
            catch (TosClientException ex)
            {
                Console.WriteLine("CompleteMultipartUpload failed, error message {0}", ex.Message);
            }
            catch (Exception ex)
            {
                Console.WriteLine("CompleteMultipartUpload failed, {0}", ex.Message);
            }
        }
    }
}

列举已上传分片

以下代码用于列举存储桶中指定对象已上传的分片信息。

using System;
using TOS;
using TOS.Error;
using TOS.Model;

namespace ConsoleApp
{
    internal class Program
    {
        private static void Main(string[] args)
        {
            var ak = Environment.GetEnvironmentVariable("TOS_ACCESS_KEY");
            var sk = Environment.GetEnvironmentVariable("TOS_SECRET_KEY");
            // endpoint 若没有指定HTTP协议（HTTP/HTTPS），默认使用 HTTPS
            // Bucket 的 Endpoint，以华北2（北京）为例：https://tos-cn-beijing.volces.com
            var endpoint = "https://tos-cn-beijing.volces.com";
            var region = "cn-beijing";
            // 填写 BucketName
            var bucketName = "*** Provide your bucket name ***";
            // 填写对象名
            var objectKey = "*** Provide your object key ***";
            // 填写分片对象对应的upload id
            var uploadID = "*** Provide upload ID ***";

            // 创建TOSClient实例
            var client = TosClientBuilder.Builder().SetAk(ak).SetSk(sk).SetEndpoint(endpoint).SetRegion(region).Build();

            try
            {
                // 列举 uploadID 已上传分片信息
                var truncated = true;
                var marker = 0;
                while (truncated)
                {
                    var createMultipartUploadInput = new ListPartsInput
                    {
                        Bucket = bucketName,
                        Key = objectKey,
                        UploadID = uploadID,
                        PartNumberMarker = marker
                    };
                    var listPartsOutput = client.ListParts(createMultipartUploadInput);
                    truncated = listPartsOutput.IsTruncated;
                    marker = listPartsOutput.NextPartNumberMarker;
                    Console.WriteLine("ListParts succeeded, request id {0}", listPartsOutput.RequestID);
                    Console.WriteLine("ListParts succeeded, upload id {0}", uploadID);
                    foreach (var part in listPartsOutput.Parts)
                    {
                        Console.WriteLine("ListParts succeeded, part number {0}", part.PartNumber);
                        Console.WriteLine("ListParts succeeded, part Etag {0}", part.ETag);
                        Console.WriteLine("ListParts succeeded, part size {0}", part.Size);
                    }
                }
            }
            catch (TosServerException ex)
            {
                Console.WriteLine("ListParts failed, request id {0}", ex.RequestID);
                Console.WriteLine("ListParts failed, status code {0}", ex.StatusCode);
                Console.WriteLine("ListParts failed, response error code {0}", ex.Code);
                Console.WriteLine("ListParts failed, response error message {0}", ex.Message);
            }
            catch (TosClientException ex)
            {
                Console.WriteLine("ListParts failed, error message {0}", ex.Message);
            }
            catch (Exception ex)
            {
                Console.WriteLine("ListParts failed, {0}", ex.Message);
            }
        }
    }
}

取消分片上传任务

您可以通过 AbortMultipartUpload 接口取消分片上传任务。当一个分片任务被取消后， TOS 会将已上传的分片数据删除，同时您无法再对此分片任务进行任何操作。
以下代码用于取消分片上传任务。

using System;
using TOS;
using TOS.Error;
using TOS.Model;

namespace ConsoleApp
{
    internal class Program
    {
        private static void Main(string[] args)
        {
            var ak = Environment.GetEnvironmentVariable("TOS_ACCESS_KEY");
            var sk = Environment.GetEnvironmentVariable("TOS_SECRET_KEY");
            // endpoint 若没有指定HTTP协议（HTTP/HTTPS），默认使用 HTTPS
            // Bucket 的 Endpoint，以华北2（北京）为例：https://tos-cn-beijing.volces.com
            var endpoint = "https://tos-cn-beijing.volces.com";
            var region = "cn-beijing";
            // 填写 BucketName
            var bucketName = "*** Provide your bucket name ***";
            // 填写对象名
            var objectKey = "*** Provide your object key ***";
            // 填写分片对象对应的upload id
            var uploadID = "*** Provide upload ID ***";

            // 创建TOSClient实例
            var client = TosClientBuilder.Builder().SetAk(ak).SetSk(sk).SetEndpoint(endpoint).SetRegion(region).Build();

            try
            {
                // 取消分片上传
                var abortMultipartUploadInput = new AbortMultipartUploadInput()
                {
                    Bucket = bucketName,
                    Key = objectKey,
                    UploadID = uploadID,
                };
                var abortMultipartUploadOutput = client.AbortMultipartUpload(abortMultipartUploadInput);
                Console.WriteLine("AbortMultipartUpload succeeded, request id {0}", abortMultipartUploadOutput.RequestID);
            }
            catch (TosServerException ex)
            {
                Console.WriteLine("AbortMultipartUpload failed, request id {0}", ex.RequestID);
                Console.WriteLine("AbortMultipartUpload failed, status code {0}", ex.StatusCode);
                Console.WriteLine("AbortMultipartUpload failed, response error code {0}", ex.Code);
                Console.WriteLine("AbortMultipartUpload failed, response error message {0}", ex.Message);
            }
            catch (TosClientException ex)
            {
                Console.WriteLine("AbortMultipartUpload failed, error message {0}", ex.Message);
            }
            catch (Exception ex)
            {
                Console.WriteLine("AbortMultipartUpload failed, {0}", ex.Message);
            }
        }
    }
}