ceph_rgw_meta_data

发表于 | 分类于

线程池入口

如下图所示，从rgw_main.cc 


进行异步操作的封装均对RGWAsyncRadosRequest进行相关操作进行重写

对RGWAsyncRadosProcess进行深入

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
//rgw_cr_rados.h
class RGWAsyncRadosProcessor{
    deque<RGWAsyncRadosRequest* > m_req_queue;
    std::atomic<bool> going_down = { false };
protected:
    RGWRados *store;
    ThreadPool m_tp;
    Throttle      req_throttle;
    
    struct RGWWQ : public ThreadPool::WorkQueue<RGWAsyncRadosRequest>{
        RGWAsyncRadosProcessor *processor;
        RGWWQ(RGWAsyncRadosProcessor *p, time_t time_out, time_t suicide_timeout, ThreadPool *tp):
            ThreadPool::WorkQueue<RGWAsyncRadosRequest>("RGWWQ", timeout, suicide_timeout, tp), processor(p){
            }
        
        bool _enqueue(RGWAsyncRadosRequest* req) override;
        void _dequeue(RGWAsyncRadosRequest* req) override{
            ceph_abort();
        }
        bool _empty() ovrride;
        RGWAsyncRadosRequest *_dequeue() override;
        using ThreadPool::WorkQueue<RGWAsyncRadosRequest>::_process;
        void _process(RGWAsyncRadosRequest *req, ThreadPool::TPHandle& handle) override;
        void _dump_queue();
        void _clear() override{
            ceph_assert(processor->m_req_queue.empty());
        }req_wq;
    }
public:
        RGWAsyncRadosProcessor(RGWRados* _store, int num_threads);
        ~RGWAsyncRadosProcessor(){}
        void start();
        void stop();
        void handle_request(RGWAsyncRadosRequest* req);
        void queue(RGWAsyncRadosRequest* req);
        bool is_going_down(){
            return going_down;
        }   
}

  • 使用工作队列进行封装 【struct RGWWQ : public ThreadPool::WorkQueue】
  • 关注_process()即可
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
RGWRemoteMetaLog
    -> RGWCoroutinesManager 
// RGWRemoteMetaLog::run_sync
int RGWRemoteMetaLog::run_sync(){
    //metadata log信息
    rgw_mdlog_info mdlog_info;
    r = read_log_info(&mdlog_info);

    //metadata 同步状态
    rgw_meta_sync_status sync_status;
    r = run(new RGWReadSyncStatusCoroutine(&sync_env, &sync_status));
    //master zone自身进行period判断，并与同步状态信息进行比较，设定初始化状态；

    //是否可进行全量同步(full sync)
}

RGW 数据存放规则

zone级别的placement

placement rule即数据的存放规则，可在placement rule中定义存储桶索引数据存放的存储

池index pool， 对象数据存放的存储池data pool中，以及通过Multipart 上传大文件临时数据
存放的存储池data extra pool 

placement rule是针对所使用的存储池进行定义，而存储池是位于zone 之下的概念，所以在

RGW中将 placement rule 作为一个 zone 级别的配置，其作用影响的力度为存储桶级，即可指定
存储桶所使用的placement rule, 那所有上传到该存储桶中的对象数据都会按照该存储桶的place
rule 定义的存储规则进行存放。用户为不同的存储桶配置不同的placement rule来实现将不同存
储桶中的对象数据存放在不同的存储节点或使用不同的存储策略；

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
//查看当前placement rule
[root@node122 src]# radosgw-admin  zone placement list --rgw-zone=default
[
    {
        "key": "default-placement",
        "val": {
            "index_pool": "default.rgw.buckets.index",
            "data_pool": "default.rgw.buckets.data",
            "data_extra_pool": "default.rgw.buckets.non-ec",
            "index_type": 0,
            "compression": ""
        }
    }
]

//添加placement rule
radosgw-admin zone placement add --rgw-zone=default --placement-id=test --index_pool=default.rgw.buckets.index --data_pool=default.rgw.buckets.data.tst --data_extra_pool=default.rgw.buckets.non-ec
{
    "id": "fea1d13c-db14-48b0-9c69-8e7662d64202",
    "name": "default",
    "domain_root": "default.rgw.meta:root",
    "control_pool": "default.rgw.control",
    "gc_pool": "default.rgw.log:gc",
    "lc_pool": "default.rgw.log:lc",
    "log_pool": "default.rgw.log",
    "intent_log_pool": "default.rgw.log:intent",
    "usage_log_pool": "default.rgw.log:usage",
    "reshard_pool": "default.rgw.log:reshard",
    "user_keys_pool": "default.rgw.meta:users.keys",
    "user_email_pool": "default.rgw.meta:users.email",
    "user_swift_pool": "default.rgw.meta:users.swift",
    "user_uid_pool": "default.rgw.meta:users.uid",
    "system_key": {
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "default.rgw.buckets.index",
                "data_pool": "default.rgw.buckets.data",
                "data_extra_pool": "default.rgw.buckets.non-ec",
                "index_type": 0,
                "compression": ""
            }
        },
        {
            "key": "test",
            "val": {
                "index_pool": "default.rgw.buckets.index",
                "data_pool": "default.rgw.buckets.data.tst",
                "data_extra_pool": "default.rgw.buckets.non-ec",
                "index_type": 0,
                "compression": ""
            }
        }
    ],
    "metadata_heap": "",
    "tier_config": [],
    "realm_id": ""
}

//也可以通过导出方式来添加
# radosgw-admin zone get --rgw-zone=default > zone.info 
# 编辑placement
# radosgw-admin zone set --rgw-zone=default < zone.info

zonegroup 级placement

1
2
3
4
5
6
7
8
9
10
11
12
13
14
//查看当前zonegroup 的placement
[root@node122 src]# radosgw-admin zonegroup placement list
[
    {
        "key": "default-placement",
        "val": {
            "name": "default-placement",
            "tags": []
        }
    }
]

//zonegroup 添加新的placement
# radosgw-admin zonegroup placement add --rgw-zonegroup=default --placement-id=test