分片上传（C SDK）--对象存储-火山引擎

文档中心

立即注册

导航

分片上传（C SDK）

最近更新时间：2024.12.26 14:43:03首次发布时间：2024.12.25 16:21:33

如果需要上传较大的对象，建议分成多个数据块（part）来分别上传，最后调用合并分片将上传的数据块合并为一个对象。

注意事项

分片上传前，您需要确保账号拥有 tos:PutObject 权限，具体操作，请参见权限配置指南。
取消分片上传任务时，您必须具备 tos:AbortMultipartUpload 权限，具体操作,请参见权限配置指南。
上传对象时，对象名必须满足命名规范，详细信息，请参见对象的命名规范。
TOS 是面向海量存储设计的分布式对象存储产品，内部分区存储了对象索引数据，为横向扩展您上传对象和下载对象时的最大吞吐量，和减小热点分区的概率，请您避免使用字典序递增的对象命名方式，详细信息,请参见性能优化。
如果桶中已经存在同名对象，则新对象会覆盖已有的对象。桶开启多版本的场景下，则会保留原有对象，生成一个新版本号用于标识新上传的对象。

分片上传步骤

使用 TOS C SDK 进行分片上传包含以下三个步骤。

初始化分片上传任务
调用 createMultipartUpload 接口返回 TOS 创建的全局唯一的uploadId。
上传分片
调用 uploadPart 接口上传分片数据。
说明
- 对于同一个分片上传任务（通过 uploadId 标识），分片编号（partNumber）标识了该分片在整个对象中的相对位置。若通过同一分片编号多次上传数据，TOS 会覆盖已有的数据，并以最后一次上传的数据为准。
- uploadPart 接口返回分片数据的 MD5 值，可通过 ETag 字段获取。合并分片时，您需指定当前分片上传任务中已上传的所有分片信息（分片编号、ETag值）。
完成分片上传
所有分片上传完成后，调用 completeMultipartUpload 接口将所有分片合并成一个完整的对象。

示例代码

初始化客户端

char* ak = getenv("TOS_ACCESS_KEY");
char* sk = getenv("TOS_SECRET_KEY");
char* sts_token = getenv("TOS_SECRET_TOKEN");

void init_test_config_ak_sk(tos_config_t *config) {
    tos_str_set(&config->endpoint, TEST_ENDPOINT);
    tos_str_set(&config->region, TEST_REGION);
    tos_str_set(&config->access_key_id, ak);
    tos_str_set(&config->access_key_secret, sk);
}

void init_test_config_sts_token(tos_config_t *config) {
    tos_str_set(&config->endpoint, TEST_ENDPOINT);
    tos_str_set(&config->region, TEST_REGION);
    tos_str_set(&config->access_key_id, ak);
    tos_str_set(&config->access_key_secret, sk);
    tos_str_set(&config->sts_token, sts_token);
}

void init_test_tos_client(tos_client_t *client) {
    client->config = tos_config_create(client->pool);

    init_test_config_ak_sk(client->config);
    // init_test_config_sts_token(client->config);

    client->ctl = tos_http_controller_create(client->pool, 0);
}

分片上传完整过程

以下代码用于将本地文件通过分片的方式上传完整过程，并在上传时指定 ACL 为 Private、存储类型为低频存储。

分片上传 From File

int test_multipart_upload_from_file() {
    // basic input
    tos_pool_t *p = NULL;
    tos_string_t bucket;
    tos_string_t object;
    tos_client_t *client = NULL;
    tos_error_t *error = NULL;

    // create multipart upload input
    create_multipart_upload_input_t *create_multipart_upload_input = NULL;
    create_multipart_upload_output_t *create_multipart_upload_output = NULL;

    // multipart upload from file input
    upload_part_from_file_input_t *upload_part_from_file_input = NULL;
    upload_part_from_file_output_t *upload_part_from_file_output = NULL;
    tos_file_buf_t *fb = NULL;
    int64_t file_length = 0;
    int64_t pos = 0;
    int part_num = 1;

    // list parts
    list_parts_input_t *list_part_input = NULL;
    list_parts_output_t *list_part_output = NULL;
    upload_part_t *part_content = NULL;

    // complete multipart upload input
    complete_multipart_upload_input_t  *complete_multipart_upload_input = NULL;
    complete_multipart_upload_output_t  *complete_multipart_upload_output = NULL;
    upload_part_t *complete_part_content = NULL;

    tos_pool_create(&p, NULL);
    client = tos_client_create(p);
    init_test_tos_client(client);

    tos_str_set(&bucket, TEST_BUCKET_NAME);
    tos_str_set(&object, TEST_MULTIPART_OBJECT);

    // create multipart upload
    if (create_multipart_upload_input_new(client->pool, &create_multipart_upload_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    tos_str_set(&create_multipart_upload_input->bucket, bucket.data);
    tos_str_set(&create_multipart_upload_input->key, object.data);
    create_multipart_upload_input->acl = ACL_PRIVATE;
    create_multipart_upload_input->storage_class = STORAGE_CLASS_IA;

    error = create_multipart_upload(client, create_multipart_upload_input, &create_multipart_upload_output);
    if (error == NULL) {
        printf("Init multipart upload succeeded, upload_id:%.*s\n",
               create_multipart_upload_output->upload_id.len,
               create_multipart_upload_output->upload_id.data);
    } else {
        printf("Init multipart upload failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);

        tos_pool_destroy(p);
        return -1;
    }

    // upload part from file
    if((fb = tos_create_file_buf(p)) == NULL)
    {
        tos_pool_destroy(p);
        return -1;
    }
    if (tos_open_file_for_all_read(p, TEST_FILE_PATH, fb) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }

    file_length = fb->file_last;
    apr_file_close(fb->file);

    if (upload_part_from_file_input_new(client->pool, &upload_part_from_file_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }

    tos_str_set(&upload_part_from_file_input->upload_id, create_multipart_upload_output->upload_id.data);
    tos_str_set(&upload_part_from_file_input->bucket, TEST_BUCKET_NAME);
    tos_str_set(&upload_part_from_file_input->key, TEST_MULTIPART_OBJECT);
    tos_str_set(&upload_part_from_file_input->file_path, TEST_FILE_PATH);

    while (pos < file_length) {
        upload_part_from_file_input->part_number = part_num++;
        upload_part_from_file_input->offset = pos;
        upload_part_from_file_input->part_size = pos + 10 * 1024 * 1024 > file_length ? file_length - pos : 10 * 1024 * 1024;
        pos += upload_part_from_file_input->part_size;
        error = upload_part_from_file(client, upload_part_from_file_input, &upload_part_from_file_output);
        if (error == NULL) {
            printf("Multipart upload part from file succeeded\n");
        } else {
            printf("Multipart upload part from file failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
                   error->request_id.len, error->request_id.data,
                   error->status_code,
                   error->message.len, error->message.data,
                   error->ec.len, error->ec.data);

            tos_pool_destroy(p);
            return -1;
        }
    }

    // list part
    if (list_parts_input_new(client->pool,&list_part_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    list_part_input->max_parts = 1000;
    tos_str_set(&list_part_input->bucket,bucket.data);
    tos_str_set(&list_part_input->key,object.data);
    tos_str_set(&list_part_input->upload_id,create_multipart_upload_output->upload_id.data);

    error = list_parts(client,list_part_input,&list_part_output);
    if (error == NULL) {
        printf("List multipart succeeded\n");
        tos_list_for_each_entry(upload_part_t, part_content, &list_part_output->parts, node) {
            printf("part_number = %d, size = %ld, last_modified = %s, etag = %s\n",
                   part_content->part_number,
                   part_content->size,
                   part_content->last_modified.data,
                   part_content->etag.data);
        }
    } else {
        printf("List multipart failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);
        tos_pool_destroy(p);
        return -1;
    }

    // complete multipart upload input
    if (complete_multipart_upload_input_new(client->pool,&complete_multipart_upload_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    tos_list_for_each_entry(upload_part_t, part_content, &list_part_output->parts, node) {
        complete_part_content = create_list_parts_content(p);
        complete_part_content->part_number = part_content->part_number;
        tos_str_set(&complete_part_content->etag, part_content->etag.data);
        tos_list_add_tail(&complete_part_content->node, &complete_multipart_upload_input->parts);
    }
    tos_str_set(&complete_multipart_upload_input->bucket,bucket.data);
    tos_str_set(&complete_multipart_upload_input->key,object.data);
    tos_str_set(&complete_multipart_upload_input->upload_id,create_multipart_upload_output->upload_id.data);

    error = complete_multipart_upload(client,complete_multipart_upload_input,&complete_multipart_upload_output);
    if (error == NULL) {
        printf("Complete multipart upload from file succeeded, key:%.*s\n",
               complete_multipart_upload_output->key.len, complete_multipart_upload_output->key.data);
    } else {
        printf("Complete multipart upload from file failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);

        tos_pool_destroy(p);
        return -1;
    }

    tos_pool_destroy(p);
    return 0;
}

分片上传 From Buffer

int test_multipart_upload_from_buffer() {
    // basic input
    tos_pool_t *p = NULL;
    tos_string_t bucket;
    tos_string_t object;
    tos_client_t *client = NULL;
    tos_error_t *error = NULL;

    // create multipart upload input
    create_multipart_upload_input_t *create_multipart_upload_input = NULL;
    create_multipart_upload_output_t *create_multipart_upload_output = NULL;

    // multipart upload from buffer input
    upload_part_from_buffer_input_t *upload_part_from_buffer_input = NULL;
    upload_part_from_buffer_output_t *upload_part_from_buffer_output = NULL;

    // list parts
    list_parts_input_t *list_part_input = NULL;
    list_parts_output_t *list_part_output = NULL;
    upload_part_t *part_content = NULL;

    // complete multipart upload input
    complete_multipart_upload_input_t  *complete_multipart_upload_input = NULL;
    complete_multipart_upload_output_t  *complete_multipart_upload_output = NULL;
    upload_part_t *complete_part_content = NULL;

    tos_pool_create(&p, NULL);
    client = tos_client_create(p);
    init_test_tos_client(client);

    tos_str_set(&bucket, TEST_BUCKET_NAME);
    tos_str_set(&object, TEST_MULTIPART_OBJECT);

    // create multipart upload
    if (create_multipart_upload_input_new(client->pool, &create_multipart_upload_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    tos_str_set(&create_multipart_upload_input->bucket, bucket.data);
    tos_str_set(&create_multipart_upload_input->key, object.data);
    create_multipart_upload_input->acl = ACL_PRIVATE;
    create_multipart_upload_input->storage_class = STORAGE_CLASS_IA;

    error = create_multipart_upload(client, create_multipart_upload_input, &create_multipart_upload_output);
    if (error == NULL) {
        printf("Init multipart upload succeeded, upload_id:%.*s\n",
               create_multipart_upload_output->upload_id.len,
               create_multipart_upload_output->upload_id.data);
    } else {
        printf("Init multipart upload failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);

        tos_pool_destroy(p);
        return -1;
    }

    for (int i = 1; i <= 3;++i)
    {
        // 分配内存
        char *str = (char *)tos_pcalloc(client->pool, 10*1024*1024);
        if (str == NULL) {
            printf(stderr, "Memory allocation failed\n");
            return -1;
        }
        // 填充字符串，例：用字符 'A' 填充
        memset(str, 'A', 10*1024*1024 - 1);
        str[10*1024*1024 - 1] = '\0'; // 确保字符串以 '\0' 结尾
        // buffer 数据
        tos_list_t buffer;
        tos_buf_t* content = NULL;
        tos_list_init(&buffer);
        content = tos_buf_pack(client->pool, str, strlen(str));
        tos_list_add_tail(&content->node, &buffer);

        // upload part from buffer
        if (upload_part_from_buffer_input_new(client->pool, &upload_part_from_buffer_input) != TOSE_OK)
        {
            tos_pool_destroy(p);
            return -1;
        }
        upload_part_from_buffer_input->content = &buffer;
        tos_str_set(&upload_part_from_buffer_input->upload_id,create_multipart_upload_output->upload_id.data);
        tos_str_set(&upload_part_from_buffer_input->bucket,bucket.data);
        tos_str_set(&upload_part_from_buffer_input->key,object.data);
        upload_part_from_buffer_input->part_number = i;

        error = upload_part_from_buffer(client,upload_part_from_buffer_input,&upload_part_from_buffer_output);
        if (error == NULL) {
            printf("Multipart upload part from buffer succeeded\n");
        } else {
            printf("Multipart upload part from buffer failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
                   error->request_id.len, error->request_id.data,
                   error->status_code,
                   error->message.len, error->message.data,
                   error->ec.len, error->ec.data);
            tos_pool_destroy(p);
            return -1;
        }
    }

    // list part
    if (list_parts_input_new(client->pool,&list_part_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    list_part_input->max_parts = 1000;
    tos_str_set(&list_part_input->bucket,bucket.data);
    tos_str_set(&list_part_input->key,object.data);
    tos_str_set(&list_part_input->upload_id,create_multipart_upload_output->upload_id.data);

    error = list_parts(client,list_part_input,&list_part_output);
    if (error == NULL) {
        printf("List multipart succeeded\n");
        tos_list_for_each_entry(upload_part_t, part_content, &list_part_output->parts, node) {
            printf("part_number = %d, size = %ld, last_modified = %s, etag = %s\n",
                   part_content->part_number,
                   part_content->size,
                   part_content->last_modified.data,
                   part_content->etag.data);
        }
    } else {
        printf("List multipart failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);
        tos_pool_destroy(p);
        return -1;
    }

    // complete multipart upload input
    if (complete_multipart_upload_input_new(client->pool,&complete_multipart_upload_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    tos_list_for_each_entry(upload_part_t, part_content, &list_part_output->parts, node) {
        complete_part_content = create_list_parts_content(p);
        complete_part_content->part_number = part_content->part_number;
        tos_str_set(&complete_part_content->etag, part_content->etag.data);
        tos_list_add_tail(&complete_part_content->node, &complete_multipart_upload_input->parts);
    }
    tos_str_set(&complete_multipart_upload_input->bucket,bucket.data);
    tos_str_set(&complete_multipart_upload_input->key,object.data);
    tos_str_set(&complete_multipart_upload_input->upload_id,create_multipart_upload_output->upload_id.data);

    error = complete_multipart_upload(client,complete_multipart_upload_input,&complete_multipart_upload_output);
    if (error == NULL) {
        printf("Complete multipart upload from file succeeded, key:%.*s\n",
               complete_multipart_upload_output->key.len, complete_multipart_upload_output->key.data);
    } else {
        printf("Complete multipart upload from file failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);

        tos_pool_destroy(p);
        return -1;
    }

    tos_pool_destroy(p);
    return 0;
}

列举已上传分片

以下代码用于列举存储桶中指定对象已上传的分片信息。

int list_parts_demo() {
    // basic input
    tos_pool_t *p = NULL;
    tos_string_t bucket;
    tos_string_t object;
    tos_client_t *client = NULL;
    tos_error_t *error = NULL;
    char* upload_id = "<fill_your_upload_id>";

    // list parts
    list_parts_input_t *list_part_input = NULL;
    list_parts_output_t *list_part_output = NULL;
    upload_part_t *part_content = NULL;

    tos_pool_create(&p, NULL);
    client = tos_client_create(p);
    init_test_tos_client(client);

    tos_str_set(&bucket, TEST_BUCKET_NAME);
    tos_str_set(&object, TEST_MULTIPART_OBJECT);

    // list part
    if (list_parts_input_new(client->pool,&list_part_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    list_part_input->max_parts = 1000;
    tos_str_set(&list_part_input->bucket,bucket.data);
    tos_str_set(&list_part_input->key,object.data);
    tos_str_set(&list_part_input->upload_id,upload_id);

    error = list_parts(client,list_part_input,&list_part_output);
    if (error == NULL) {
        printf("List multipart succeeded\n");
        tos_list_for_each_entry(upload_part_t, part_content, &list_part_output->parts, node) {
            printf("part_number = %d, size = %ld, last_modified = %s, etag = %s\n",
                   part_content->part_number,
                   part_content->size,
                   part_content->last_modified.data,
                   part_content->etag.data);
        }
    } else {
        printf("List multipart failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);
        tos_pool_destroy(p);
        return -1;
    }

    tos_pool_destroy(p);
    return 0;
}

取消分片上传任务

您可以通过 AbortMultipartUpload 接口取消分片上传任务。当一个分片任务被取消后， TOS 会将已上传的分片数据删除，同时您无法再对此分片任务进行任何操作。
以下代码用于取消分片上传任务。

int abort_multipart_upload_demo()
{
    // basic input
    tos_pool_t *p = NULL;
    tos_string_t bucket;
    tos_string_t object;
    tos_client_t *client = NULL;
    tos_error_t *error = NULL;
    char* upload_id = "<fill_your_upload_id>";

    // abort abort multipart upload input
    abort_multipart_upload_input_t *abort_multipart_upload_input = NULL;
    abort_multipart_upload_output_t *abort_multipart_upload_output = NULL;

    tos_pool_create(&p, NULL);
    client = tos_client_create(p);
    init_test_tos_client(client);

    tos_str_set(&bucket, TEST_BUCKET_NAME);
    tos_str_set(&object, TEST_MULTIPART_OBJECT);
    
        // abort multipart upload
    if (abort_multipart_upload_input_new(client->pool, &abort_multipart_upload_input) != TOSE_OK)
    {
        tos_pool_destroy(p);
        return -1;
    }
    tos_str_set(&abort_multipart_upload_input->bucket, TEST_BUCKET_NAME);
    tos_str_set(&abort_multipart_upload_input->key, TEST_MULTIPART_OBJECT);
    tos_str_set(&abort_multipart_upload_input->upload_id, "<fill_your_upload_id>");
        error = abort_multipart_upload(client, abort_multipart_upload_input, &abort_multipart_upload_output);
    if (error == NULL) {
        printf("Abort multipart upload succeeded, upload_id:%s\n", upload_id);
    } else {
        printf("Abort multipart upload failed, request_id:%.*s, status_code:%d, message:%.*s, ec:%.*s\n",
               error->request_id.len, error->request_id.data,
               error->status_code,
               error->message.len, error->message.data,
               error->ec.len, error->ec.data);
        tos_pool_destroy(p);
        return -1;
    }

    tos_pool_destroy(p);
    return 0;
}

对象存储