PostgreSQL的vacuum过程中heap_vacuum_rel函数分析

这篇文章主要介绍“postgresql的vacuum过程中heap_vacuum_rel函数分析”，在日常操作中，相信很多人在Postgresql的vacuum过程中heap_vacuum_rel函数分析问题上存在疑惑，小编查阅了各式资料，整理出简单好用的操作方法，希望对大家解答”PostgreSQL的vacuum过程中heap_vacuum_rel函数分析”的疑惑有所帮助！接下来，请跟着小编一起来学习吧！

本节简单介绍了PostgreSQL手工执行vacuum的实现逻辑,主要分析了ExecVacuum->vacuum->vacuum_rel->heap_vacuum_rel函数的实现逻辑。

一、数据结构

宏定义
Vacuum和Analyze命令选项

typedef enum VacuumOption
{
    VACOPT_VACUUM = 1 << 0,     
    VACOPT_ANALYZE = 1 << 1,    
    VACOPT_VERBOSE = 1 << 2,    
    VACOPT_FREEZE = 1 << 3,     
    VACOPT_FULL = 1 << 4,       
    VACOPT_SKIP_LOCKED = 1 << 5,    
    VACOPT_SKIPTOAST = 1 << 6,  
    VACOPT_DISABLE_PAGE_SKIPPING = 1 << 7   
} VacuumOption;

VacuumStmt
存储vacuum命令的option&Relation链表

typedef struct VacuumStmt
{
    nodeTag     type;//Tag
    //VacuumOption位标记
    int         options;        
    //VacuumRelation链表,如为NIL-->所有Relation.
    List       *rels;           
} VacuumStmt;

VacuumParams
vacuum命令参数

typedef struct VacuumParams
{
    //最小freeze age,-1表示使用默认
    int         freeze_min_age; 
    //扫描整个table的freeze age
    int         freeze_table_age;   
    //最小的multixact freeze age,-1表示默认
    int         multixact_freeze_min_age;   
    //扫描全表的freeze age,-1表示默认
    int         multixact_freeze_table_age; 
    //是否强制wraparound?
    bool        is_wraparound;  
    //以毫秒为单位的最小执行阈值
    int         log_min_duration;   
} VacuumParams;

VacuumRelation
VACUUM/ANALYZE命令的目标表信息

typedef struct VacuumRelation
{
    NodeTag     type;
    RangeVar   *relation;       
    Oid         oid;            
    List       *va_cols;        
} VacuumRelation;

BufferAccessStrategy
Buffer访问策略对象

typedef int Buffer;
#define InvalidBuffer   0

typedef struct BufferAccessStrategyData *BufferAccessStrategy;
 
typedef struct BufferAccessStrategyData
{
    
    //全局的策略类型
    BufferAccessStrategyType btype;
    
    //buffers[]中的元素个数
    int         ring_size;
    
    int         current;
    
    bool        current_was_in_ring;
    
    Buffer      buffers[FLEXIBLE_ARRAY_MEMBER];
}           BufferAccessStrategyData;
//Block结构体指针
typedef void *Block;

//GetAccessStrategy()函数可取值的参数
typedef enum BufferAccessStrategyType
{
    //常规的随机访问
    BAS_NORMAL,                 
    //大规模的只读扫描
    BAS_BULKREAD,               
    //大量的多块写(如 COPY IN)
    BAS_BULKWRITE,              
    //VACUUM
    BAS_VACUUM                  
} BufferAccessStrategyType;

LVRelStats

typedef struct LVRelStats
{
    
    //T表示two-pass strategy,F表示one-pass strategy
    bool        hasindex;
    
    //rel的全局统计信息
    //pg_class.relpages的上一个值
    BlockNumber old_rel_pages;  
    //pages的总数
    BlockNumber rel_pages;      
    //扫描的pages
    BlockNumber scanned_pages;  
    //由于pin跳过的pages
    BlockNumber pinskipped_pages;   
    //跳过的frozen pages
    BlockNumber frozenskipped_pages;    
    //计算其元组的pages
    BlockNumber tupcount_pages; 
    //pg_class.reltuples的前值
    double      old_live_tuples;    
    //新估算的总元组数
    double      new_rel_tuples; 
    //新估算的存活元组数
    double      new_live_tuples;    
    //新估算的废弃元组数
    double      new_dead_tuples;    
    //已清除的pages
    BlockNumber pages_removed;
    //已删除的tuples
    double      tuples_deleted;
    //实际上是非空page + 1
    BlockNumber nonempty_pages; 
    
    
    //将要删除的元组TIDs链表
    //注意:该链表已使用TID地址排序
    //当前的入口/条目数
    int         num_dead_tuples;    
    //数组中已分配的slots(最大已废弃元组数)
    int         max_dead_tuples;    
    //ItemPointerData数组
    ItemPointer dead_tuples;    
    //扫描的索引数
    int         num_index_scans;
    //最后被清除的事务ID
    TransactionId latestRemovedXid;
    //是否存在waiter?
    bool        lock_waiter_detected;
} LVRelStats;

PGRUsage
pg_rusage_init/pg_rusage_show的状态结构体

//pg_rusage_init/pg_rusage_show的状态结构体
typedef struct PGRUsage
{
    struct timeval tv;
    struct rusage ru;
} PGRUsage;
struct rusage
{
    struct timeval ru_utime;    
    struct timeval ru_stime;    
};
struct timeval
{
__time_t tv_sec;        
__suseconds_t tv_usec;  
};

二、源码解读

heap_vacuum_rel() — 为heap relation执行VACUUM
大体逻辑如下:
1.初始化相关变量,如本地变量/日志记录级别/访问策略等
2.调用vacuum_set_xid_limits计算最老的xmin和冻结截止点
3.判断是否执行全表(不跳过pages)扫描,标记变量为aggressive
4.初始化统计信息结构体vacrelstats
5.打开索引,执行函数lazy_scan_heap进行vacuuming,关闭索引
6.更新pg_class中的统计信息
7.收尾工作

void
heap_vacuum_rel(Relation onerel, int options, VacuumParams *params,
                BufferAccessStrategy bstrategy)
{
    LVRelStats *vacrelstats;//统计信息
    Relation   *Irel;//关系指针
    int         nindexes;
    PGRUsage    ru0;//状态结构体
    TimestampTz starttime = 0;//时间戳
    long        secs;//秒数
    int         usecs;//微秒数
    double      read_rate,//读比率
                write_rate;//写比率
    //是否扫描所有未冻结的pages?
    bool        aggressive;     
    //实际上是否扫描了所有这样的pages?
    bool        scanned_all_unfrozen;   
    TransactionId xidFullScanLimit;
    MultiXactId mxactFullScanLimit;
    BlockNumber new_rel_pages;
    BlockNumber new_rel_allvisible;
    double      new_live_tuples;
    TransactionId new_frozen_xid;
    MultiXactId new_min_multi;
    Assert(params != NULL);
    
    //如autovacuum日志记录需要,则测量耗费的时间
    if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0)
    {
        pg_rusage_init(&ru0);
        starttime = GetCurrentTimestamp();
    }
    if (options & VACOPT_VERBOSE)
        //需要VERBOSE
        elevel = INFO;
    else
        elevel = DEBUG2;
    pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
                                  RelationGetRelid(onerel));
    vac_strategy = bstrategy;
    //计算最老的xmin和冻结截止点
    //输出:OldestXmin/FreezeLimit/FreezeLimit/MultiXactCutoff/mxactFullScanLimit
    vacuum_set_xid_limits(onerel,
                          params->freeze_min_age,
                          params->freeze_table_age,
                          params->multixact_freeze_min_age,
                          params->multixact_freeze_table_age,
                          &OldestXmin, &FreezeLimit, &xidFullScanLimit,
                          &MultiXactCutoff, &mxactFullScanLimit);
    
    //比较onerel->rd_rel->relfrozenxid & xidFullScanLimit
    //如小于等于,则aggressive为T,否则为F
    aggressive = TransactionIdPrecedesOrEquals(onerel->rd_rel->relfrozenxid,
                                               xidFullScanLimit);
    //比较onerel->rd_rel->relminmxid &mxactFullScanLimit
    //如小于等于,则aggressive为T
    //否则aggressive原值为T,则为T,否则为F
    aggressive |= MultiXactIdPrecedesOrEquals(onerel->rd_rel->relminmxid,
                                              mxactFullScanLimit);
    if (options & VACOPT_DISABLE_PAGE_SKIPPING)
        //禁用跳过页,则强制为T
        aggressive = true;
    //分配统计结构体内存
    vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats));
    //记录统计信息
    vacrelstats->old_rel_pages = onerel->rd_rel->relpages;
    vacrelstats->old_live_tuples = onerel->rd_rel->reltuples;
    vacrelstats->num_index_scans = 0;
    vacrelstats->pages_removed = 0;
    vacrelstats->lock_waiter_detected = false;
    
    //打开该relation所有的索引
    vac_open_indexes(onerel, RowExclusiveLock, &nindexes, &Irel);
    vacrelstats->hasindex = (nindexes > 0);
    
    //执行vacuuming
    lazy_scan_heap(onerel, options, vacrelstats, Irel, nindexes, aggressive);
    
    //已完成index的处理
    vac_close_indexes(nindexes, Irel, NoLock);
    
    if ((vacrelstats->scanned_pages + vacrelstats->frozenskipped_pages)
        < vacrelstats->rel_pages)
    {
        Assert(!aggressive);
        scanned_all_unfrozen = false;
    }
    else
        //扫描pages + 冻结跳过的pages >= 总pages,则为T
        scanned_all_unfrozen = true;
    
    if (should_attempt_truncation(vacrelstats))
        lazy_truncate_heap(onerel, vacrelstats);
    
    //通知其他进程,正在进行最后的清理
    pgstat_progress_update_param(PROGRESS_VACUUM_PHASE,
                                 PROGRESS_VACUUM_PHASE_FINAL_CLEANUP);
    
    new_rel_pages = vacrelstats->rel_pages;
    new_live_tuples = vacrelstats->new_live_tuples;
    if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
    {
        //实际处理的tuple为0而且总页面不为0,则调整回原页数
        new_rel_pages = vacrelstats->old_rel_pages;
        new_live_tuples = vacrelstats->old_live_tuples;
    }
    visibilitymap_count(onerel, &new_rel_allvisible, NULL);
    if (new_rel_allvisible > new_rel_pages)
        new_rel_allvisible = new_rel_pages;
    new_frozen_xid = scanned_all_unfrozen ? FreezeLimit : InvalidTransactionId;
    new_min_multi = scanned_all_unfrozen ? MultiXactCutoff : InvalidMultiXactId;
    //更新pg_class中的统计信息
    vac_update_relstats(onerel,
                        new_rel_pages,
                        new_live_tuples,
                        new_rel_allvisible,
                        vacrelstats->hasindex,
                        new_frozen_xid,
                        new_min_multi,
                        false);
    
    //同时,发送结果给统计收集器
    pgstat_report_vacuum(RelationGetRelid(onerel),
                         onerel->rd_rel->relisshared,
                         new_live_tuples,
                         vacrelstats->new_dead_tuples);
    pgstat_progress_end_command();
    
    //并在适当的情况下记录操作
    if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0)
    {
        //autovacuum && 参数log_min_duration >= 0
        TimestampTz endtime = GetCurrentTimestamp();
        if (params->log_min_duration == 0 ||
            TimestampDifferenceExceeds(starttime, endtime,
                                       params->log_min_duration))
        {
            StringInfoData buf;
            char       *msgfmt;
            TimestampDifference(starttime, endtime, &secs, &usecs);
            read_rate = 0;
            write_rate = 0;
            if ((secs > 0) || (usecs > 0))
            {
                read_rate = (double) BLCKSZ * VacuumPageMiss / (1024 * 1024) /
                    (secs + usecs / 1000000.0);
                write_rate = (double) BLCKSZ * VacuumPageDirty / (1024 * 1024) /
                    (secs + usecs / 1000000.0);
            }
            
            initStringInfo(&buf);
            if (params->is_wraparound)
            {
                if (aggressive)
                    msgfmt = _("automatic aggressive vacuum to prevent wraparound of table \"%s.%s.%s\": index scans: %d\n");
                else
                    msgfmt = _("automatic vacuum to prevent wraparound of table \"%s.%s.%s\": index scans: %d\n");
            }
            else
            {
                if (aggressive)
                    msgfmt = _("automatic aggressive vacuum of table \"%s.%s.%s\": index scans: %d\n");
                else
                    msgfmt = _("automatic vacuum of table \"%s.%s.%s\": index scans: %d\n");
            }
            appendStringInfo(&buf, msgfmt,
                             get_database_name(MyDatabaseId),
                             get_namespace_name(RelationGetNamespace(onerel)),
                             RelationGetRelationName(onerel),
                             vacrelstats->num_index_scans);
            appendStringInfo(&buf, _("pages: %u removed, %u remain, %u skipped due to pins, %u skipped frozen\n"),
                             vacrelstats->pages_removed,
                             vacrelstats->rel_pages,
                             vacrelstats->pinskipped_pages,
                             vacrelstats->frozenskipped_pages);
            appendStringInfo(&buf,
                             _("tuples: %.0f removed, %.0f remain, %.0f are dead but not yet removable, oldest xmin: %u\n"),
                             vacrelstats->tuples_deleted,
                             vacrelstats->new_rel_tuples,
                             vacrelstats->new_dead_tuples,
                             OldestXmin);
            appendStringInfo(&buf,
                             _("buffer usage: %d hits, %d misses, %d dirtied\n"),
                             VacuumPageHit,
                             VacuumPageMiss,
                             VacuumPageDirty);
            appendStringInfo(&buf, _("avg read rate: %.3f MB/s, avg write rate: %.3f MB/s\n"),
                             read_rate, write_rate);
            appendStringInfo(&buf, _("system usage: %s"), pg_rusage_show(&ru0));
            ereport(LOG,
                    (errmsg_internal("%s", buf.data)));
            pfree(buf.data);
        }
    }
}

三、跟踪分析

测试脚本

11:45:37 (xdb@[local]:5432)testdb=# vacuum t1;

启动gdb,设置断点
注:PG主线函数名称已改为heap_vacuum_rel,PG 11.1仍为lazy_vacuum_rel

(gdb) c
Continuing.
Breakpoint 1, lazy_vacuum_rel (onerel=0x7f226cd9e9a0, options=1, params=0x7ffe010d5b70, bstrategy=0x1da9708)
    at vacuumlazy.c:197
197     TimestampTz starttime = 0;
(gdb)

输入参数
relation

(gdb) p *onerel
$1 = {rd_node = {spcNode = 1663, dbNode = 16402, relNode = 50820}, rd_smgr = 0x0, rd_refcnt = 1, rd_backend = -1, 
  rd_islocaltemp = false, rd_isnailed = false, rd_isvalid = true, rd_indexvalid = 0 '\000', rd_statvalid = false, 
  rd_createSubid = 0, rd_newRelfilenodeSubid = 0, rd_rel = 0x7f226cd9ebb8, rd_att = 0x7f226cd9ecd0, rd_id = 50820, 
  rd_lockInfo = {lockRelId = {relId = 50820, dbId = 16402}}, rd_rules = 0x0, rd_rulescxt = 0x0, trigdesc = 0x0, 
  rd_rsdesc = 0x0, rd_fkeylist = 0x0, rd_fkeyvalid = false, rd_parTKEycxt = 0x0, rd_partkey = 0x0, rd_pdcxt = 0x0, 
  rd_partdesc = 0x0, rd_partcheck = 0x0, rd_indexlist = 0x0, rd_oidindex = 0, rd_pkindex = 0, rd_replidindex = 0, 
  rd_statlist = 0x0, rd_indexattr = 0x0, rd_projindexattr = 0x0, rd_keyattr = 0x0, rd_pkattr = 0x0, rd_idattr = 0x0, 
  rd_projidx = 0x0, rd_pubactions = 0x0, rd_options = 0x0, rd_index = 0x0, rd_indextuple = 0x0, rd_amhandler = 0, 
  rd_indexcxt = 0x0, rd_amroutine = 0x0, rd_opfamily = 0x0, rd_opcintype = 0x0, rd_support = 0x0, rd_supportinfo = 0x0, 
  rd_indoption = 0x0, rd_indexprs = 0x0, rd_indpred = 0x0, rd_exclops = 0x0, rd_exclprocs = 0x0, rd_exclstrats = 0x0, 
  rd_amcache = 0x0, rd_indcollation = 0x0, rd_fdwroutine = 0x0, rd_toastoid = 0, pgstat_info = 0x1d5a030}
(gdb)

vacuum参数

(gdb) p *params
$2 = {freeze_min_age = -1, freeze_table_age = -1, multixact_freeze_min_age = -1, multixact_freeze_table_age = -1, 
  is_wraparound = false, log_min_duration = -1}
(gdb)

buffer访问策略对象

(gdb) p *bstrategy
$3 = {btype = BAS_VACUUM, ring_size = 32, current = 0, current_was_in_ring = false, buffers = 0x1da9718}
(gdb) 
(gdb) p *bstrategy->buffers
$4 = 0
(gdb)

1.初始化相关变量,如本地变量/日志记录级别/访问策略等

$4 = 0
(gdb) n
215     if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0)
(gdb) 
221     if (options & VACOPT_VERBOSE)
(gdb) 
224         elevel = DEBUG2;
(gdb) 
226     pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
(gdb) 
229     vac_strategy = bstrategy;
(gdb)

2.调用vacuum_set_xid_limits计算最老的xmin和冻结截止点
返回值均为默认值,其中OldestXmin是当前最小的活动事务ID

231     vacuum_set_xid_limits(onerel,
(gdb) 
245     aggressive = TransactionIdPrecedesOrEquals(onerel->rd_rel->relfrozenxid,
(gdb) p OldestXmin
$5 = 307590
(gdb) p FreezeLimit
$6 = 4245274886
(gdb) p xidFullScanLimit
$10 = 4145274886
(gdb) p MultiXactCutoff
$8 = 4289967297
(gdb) p mxactFullScanLimit
$9 = 4144967297
(gdb)

3.判断是否执行全表(不跳过pages)扫描,标记变量为aggressive,值为F

(gdb) n
247     aggressive |= MultiXactIdPrecedesOrEquals(onerel->rd_rel->relminmxid,
(gdb) p aggressive
$11 = false
(gdb) n
249     if (options & VACOPT_DISABLE_PAGE_SKIPPING)
(gdb) p onerel->rd_rel->relfrozenxid
$12 = 144983
(gdb) p xidFullScanLimit
$13 = 4145274886
(gdb)

4.初始化统计信息结构体vacrelstats

(gdb) n
252     vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats));
(gdb) 
254     vacrelstats->old_rel_pages = onerel->rd_rel->relpages;
(gdb) 
255     vacrelstats->old_live_tuples = onerel->rd_rel->reltuples;
(gdb) 
256     vacrelstats->num_index_scans = 0;
(gdb) 
257     vacrelstats->pages_removed = 0;
(gdb) 
258     vacrelstats->lock_waiter_detected = false;
(gdb) 
261     vac_open_indexes(onerel, RowExclusiveLock, &nindexes, &Irel);
(gdb) p *vacrelstats
$14 = {hasindex = false, old_rel_pages = 75, rel_pages = 0, scanned_pages = 0, pinskipped_pages = 0, 
  frozenskipped_pages = 0, tupcount_pages = 0, old_live_tuples = 10000, new_rel_tuples = 0, new_live_tuples = 0, 
  new_dead_tuples = 0, pages_removed = 0, tuples_deleted = 0, nonempty_pages = 0, num_dead_tuples = 0, max_dead_tuples = 0, 
  dead_tuples = 0x0, num_index_scans = 0, latestRemovedXid = 0, lock_waiter_detected = false}
(gdb)

5.打开索引,执行函数lazy_scan_heap进行vacuuming,关闭索引

(gdb) n
262     vacrelstats->hasindex = (nindexes > 0);
(gdb) 
265     lazy_scan_heap(onerel, options, vacrelstats, Irel, nindexes, aggressive);
(gdb) 
268     vac_close_indexes(nindexes, Irel, NoLock);
(gdb) 
(gdb) 
277     if ((vacrelstats->scanned_pages + vacrelstats->frozenskipped_pages)
(gdb) 
278         < vacrelstats->rel_pages)
(gdb) 
277     if ((vacrelstats->scanned_pages + vacrelstats->frozenskipped_pages)
(gdb) 
284         scanned_all_unfrozen = true;
(gdb) p *vacrelstats
$15 = {hasindex = true, old_rel_pages = 75, rel_pages = 75, scanned_pages = 75, pinskipped_pages = 0, 
  frozenskipped_pages = 0, tupcount_pages = 75, old_live_tuples = 10000, new_rel_tuples = 10154, new_live_tuples = 10000, 
  new_dead_tuples = 154, pages_removed = 0, tuples_deleted = 0, nonempty_pages = 75, num_dead_tuples = 0, 
  max_dead_tuples = 21825, dead_tuples = 0x1db5030, num_index_scans = 0, latestRemovedXid = 0, lock_waiter_detected = false}
(gdb) p vacrelstats->scanned_pages
$16 = 75
(gdb) p vacrelstats->frozenskipped_pages
$17 = 0
(gdb) p vacrelstats->rel_pages
$18 = 75
(gdb)

6.更新pg_class中的统计信息

(gdb) n
289     if (should_attempt_truncation(vacrelstats))
(gdb) 
293     pgstat_progress_update_param(PROGRESS_VACUUM_PHASE,
(gdb) 
317     new_rel_pages = vacrelstats->rel_pages;
(gdb) 
318     new_live_tuples = vacrelstats->new_live_tuples;
(gdb) 
319     if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
(gdb) 
325     visibilitymap_count(onerel, &new_rel_allvisible, NULL);
(gdb) 
326     if (new_rel_allvisible > new_rel_pages)
(gdb) p new_rel_allvisible
$19 = 0
(gdb) p new_rel_pages
$20 = 75
(gdb) n
329     new_frozen_xid = scanned_all_unfrozen ? FreezeLimit : InvalidTransactionId;
(gdb) 
330     new_min_multi = scanned_all_unfrozen ? MultiXactCutoff : InvalidMultiXactId;
(gdb) 
336                         vacrelstats->hasindex,
(gdb) 
332     vac_update_relstats(onerel,
(gdb) p new_frozen_xid
$21 = 4245274886
(gdb) p new_min_multi
$22 = 4289967297
(gdb)

7.收尾工作

(gdb) n
345                          vacrelstats->new_dead_tuples);
(gdb) 
342     pgstat_report_vacuum(RelationGetRelid(onerel),
(gdb) 
343                          onerel->rd_rel->relisshared,
(gdb) 
342     pgstat_report_vacuum(RelationGetRelid(onerel),
(gdb) 
346     pgstat_progress_end_command();
(gdb) 
349     if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0)
(gdb)

完成调用

411 }
(gdb) 
vacuum_rel (relid=50820, relation=0x1cdb8d0, options=1, params=0x7ffe010d5b70) at vacuum.c:1560
1560        AtEOXact_GUC(false, save_nestlevel);
(gdb)

到此，关于“PostgreSQL的vacuum过程中heap_vacuum_rel函数分析”的学习就结束了，希望能够解决大家的疑惑。理论与实践的搭配能更好的帮助大家学习，快去试试吧！若想继续学习更多相关知识，请继续关注编程网网站，小编会继续努力为大家带来更多实用的文章！