58 views

XLOG文件回收和删除

By | 2019年2月25日

一、基本原理:每个检查点都将不再需要的XLOG文件回收或删除。

1。第一步是估计在两个检查点之间生成的xlog的数量。根据这个数量,可以计算出未来最大的日志文件号,然后将不需要的文件恢复并重命名为将来要使用的日志文件号。

1.1 UpdateCheckPointDistanceEstimate估算checkpoint之前产生的日志量:
if (CheckPointDistanceEstimate < nbytes)//上次估算量比这次估算的小,则更新为这次的估算量
CheckPointDistanceEstimate = nbytes;
else//否则,适当增加
CheckPointDistanceEstimate =(0.90 CheckPointDistanceEstimate + 0.10 (double) nbytes);

2、计算上一次checkpoint时,所在的文件段号_logSegNo:
XLByteToSeg(PriorRedoPtr, _logSegNo);

3、计算需要保留的文件段号:从该段号_logSegNo开始的文件都不能被删除,之前的需要删除或回收:根据备机请求以及wal_keep_segments计算KeepLogSeg(recptr, &_logSegNo);

4、遍历pg_wal目录下的所有xlog文件,进行删除:RemoveOldXlogFiles

4.1 跳过时间线进行比较,如果pg_wal目录下的文件比_logSegNo小则被删除或回收。那么什么条件下次被回收?
–RemoveXlogFile

4.2 计算回收文件重命名的未来最大文件段号recycleSegNo:
1)如果本次是第一次checkpoint,则未来最大段号recycleSegNo=当前段文件号+10

2)否则调用函数XLOGfileslop计算:
2.1 估算下一次checkpoint结束时日志位置:
distance=(2.0+checkpoint_completion_target)CheckPointDistanceEstimate
distance=1.1
recycleSegNo = (XLogSegNo) ceil(((double) PriorRedoPtr + distance) / XLOG_SEG_SIZE);
2.2 minSegNo = PriorRedoPtr / XLOG_SEG_SIZE + ConvertToXSegs(min_wal_size_mb) – 1;
maxSegNo = PriorRedoPtr / XLOG_SEG_SIZE + ConvertToXSegs(max_wal_size_mb) – 1;
2.3 if (recycleSegNo < minSegNo)
recycleSegNo = minSegNo;
if (recycleSegNo > maxSegNo)
recycleSegNo = maxSegNo;

4.3 如果当前段文件号endlogSegNo < recycleSegNo,则调用InstallXLogFileSegment进行回收:
1)在endlogSegNo和recycleSegNo之间找一个free slot num,即没有该段文件号的xlog文件

2)将需要删除的文件名命名为该free slot号的文件名

3)如果没有找到free slot则直接删除该文件   –RemoveXlogFile

二、代码流程
1、checkpoint顶层函数CreateCheckPoint:

CreateCheckPoint:
XLogCtlInsert *Insert = &XLogCtl->Insert;//标识插入的位置
curInsert = XLogBytePosToRecPtr(Insert->CurrBytePos);//添加页头大小后的位置
//(((curInsert) % XLOG_BLCKSZ == 0) ? 0 : (XLOG_BLCKSZ – (curInsert) % XLOG_BLCKSZ))
freespace = INSERT_FREESPACE(curInsert);//curInsert所在页是否有空闲空间
if (freespace == 0){
if (curInsert % XLogSegSize == 0)//正好一个xlog段文件用完,即将使用下一个段文件,则跳过36字节
curInsert += SizeOfXLogLongPHD;//36字节
else//xlog段文件中正好一页用完,即将使用下一页,则跳过20字节
curInsert += SizeOfXLogShortPHD;//20字节
}
checkPoint.redo = curInsert;//xlog文件上,实际的即将插入位置
RedoRecPtr = XLogCtl->Insert.RedoRecPtr = checkPoint.redo;

//插入checkpoint记录后末尾位置,即下一个xlog开始的位置
recptr = XLogInsert(RM_XLOG_ID,shutdown ? XLOG_CHECKPOINT_SHUTDOWN :XLOG_CHECKPOINT_ONLINE);

PriorRedoPtr = ControlFile->checkPointCopy.redo;//上一次checkpoint的起始位置

if (PriorRedoPtr != InvalidXLogRecPtr){//上一次checkpoint开始到这一次checkpoint开始,产生的XLOG大小为入参
/*
CheckPointDistanceEstimate:
1、CheckPointDistanceEstimate<RedoRecPtr – PriorRedoPtr时:RedoRecPtr – PriorRedoPtr
2、CheckPointDistanceEstimate>=RedoRecPtr – PriorRedoPtr时:0.9*CheckPointDistanceEstimate+0.1*(RedoRecPtr – PriorRedoPtr)
*/
UpdateCheckPointDistanceEstimate(RedoRecPtr – PriorRedoPtr);
//_logSegNo = (PriorRedoPtr) / XLogSegSize
XLByteToSeg(PriorRedoPtr, _logSegNo);
KeepLogSeg(recptr, &_logSegNo);
_logSegNo–;
RemoveOldXlogFiles(_logSegNo, PriorRedoPtr, recptr);

2、两个宏定义

#define UsableBytesInPage (XLOG_BLCKSZ – SizeOfXLogShortPHD)//注意:不是文件第一页
#define UsableBytesInSegment ((XLOG_SEG_SIZE / XLOG_BLCKSZ) * UsableBytesInPage – (SizeOfXLogLongPHD – SizeOfXLogShortPHD))

3、函数XLogBytePosToRecPtr

static XLogRecPtr
XLogBytePosToRecPtr(uint64 bytepos)
{

//bytepos:不包括xlog页的页头等额外字节占用的大小
fullsegs = bytepos / UsableBytesInSegment;
bytesleft = bytepos % UsableBytesInSegment;
/*
1、如果bytesleft < XLOG_BLCKSZ-32,则表示定位到第一页上,则文件偏移值跳过第一页页头大小
2、如果bytesleft >= XLOG_BLCKSZ-32,则表示定位不是第一页
*/
if (bytesleft < XLOG_BLCKSZ – SizeOfXLogLongPHD){
/* fits on first page of segment */
seg_offset = bytesleft + SizeOfXLogLongPHD;
}else{
/* account for the first page on segment with long header */
seg_offset = XLOG_BLCKSZ;//先跳过第一页
bytesleft -= XLOG_BLCKSZ – SizeOfXLogLongPHD;//去掉第一页存放XLOG的大小

fullpages = bytesleft / UsableBytesInPage;//剩下的需要几个页
bytesleft = bytesleft % UsableBytesInPage;//剩下的偏移
//  文件偏移=第一页大小+剩下的几个页大小+剩下的偏移+最后一页的页头
seg_offset += fullpages * XLOG_BLCKSZ + bytesleft + SizeOfXLogShortPHD;
}
//result=(fullsegs) * XLOG_SEG_SIZE + seg_offset
XLogSegNoOffsetToRecPtr(fullsegs, seg_offset, result);
return result;
}

4、函数KeepLogSeg

static void
KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
{
//segno为当前xlog即将插入位置在第几个文件上
XLByteToSeg(recptr, segno);
//XLogCtl->replicationSlotMinLSN;备机上请求预留的最小值?
keep = XLogGetReplicationSlotMinimumLSN();
/* compute limit for wal_keep_segments first */
if (wal_keep_segments > 0){
/*
首先计算wal_keep_segments得到的限制:
1、比如wal_keep_segments值是10,若当前insert的位置的文件号segno为5,那么向前推进到1

2、否则向前推进wal_keep_segments后的segno前的可删除
*/
if (segno <= wal_keep_segments)
segno = 1;
else
segno = segno – wal_keep_segments;
}

/* then check whether slots limit removal further */
//计算slots限制,如果其算出的值小于wal_keep_segments计算出的值,则需要使用slotSegNo,slots还有用,不能删除
if (max_replication_slots > 0 && keep != InvalidXLogRecPtr){
XLByteToSeg(keep, slotSegNo);
if (slotSegNo <= 0)
segno = 1;
else if (slotSegNo < segno)
segno = slotSegNo;
}

/* don’t delete WAL segments newer than the calculated segment */
if (segno < *logSegNo)
*logSegNo = segno;
//note:
}

5、函数RemoveOldXlogFiles
static void
RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr)
{
//首先获取xlog目录
xldir = AllocateDir(XLOGDIR);
if (xldir == NULL)
ereport(ERROR,
(errcode_for_file_access(),
errmsg(“could not open write-ahead log directory \”%s\”: %m”,
XLOGDIR)));

/*
构建一个log文件名,用于判断,该文件之前的xlog可以删除。用不到时间线,所以可以使用0
*/
XLogFileName(lastoff, 0, segno);

while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL){
/* 忽略非xlog文件 */
if (!IsXLogFileName(xlde->d_name) &&
!IsPartialXLogFileName(xlde->d_name))
continue;
/*
1、跳过时间线进行比较
*/
if (strcmp(xlde->d_name + 8, lastoff + 8) <= 0){
if (XLogArchiveCheckDone(xlde->d_name)){//如果没有开启归档:总是TRUE;否则,归档完成后才为TRUE
/* Update the last removed location in shared memory first */
//XLogCtl->lastRemovedSegNo = segno;
UpdateLastRemovedPtr(xlde->d_name);
RemoveXlogFile(xlde->d_name, PriorRedoPtr, endptr);
}
}
}

6、函数RemoveXlogFile
RemoveXlogFile(const char *segname, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr)
{

XLByteToSeg(endptr, endlogSegNo);
if (PriorRedoPtr == InvalidXLogRecPtr)
recycleSegNo = endlogSegNo + 10;
else
recycleSegNo = XLOGfileslop(PriorRedoPtr);

snprintf(path, MAXPGPATH, XLOGDIR “/%s”, segname);

if (endlogSegNo <= recycleSegNo &&
lstat(path, &statbuf) == 0 && S_ISREG(statbuf.st_mode) &&
InstallXLogFileSegment(&endlogSegNo, path,
true, recycleSegNo, true))
{
endlogSegNo++;
}else{
rc = durable_unlink(path, LOG);
}
}

7、函数InstallXLogFileSegment
static bool
InstallXLogFileSegment(XLogSegNo segno, char tmppath,
bool find_free, XLogSegNo max_segno,
bool use_lock)
{
XLogFilePath(path, ThisTimeLineID, segno);
/

We want to be sure that only one process does this at a time.
*/
if (use_lock)
LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
if (!find_free)
{
/ Force installation: get rid of any pre-existing segment file /
durable_unlink(path, DEBUG1);//删除文件并持久化到磁盘
}else{
/ Find a free slot to put it in /
while (stat(path, &stat_buf) == 0){//获取文件信息并保存到stat_buf中,成功返回0
//在segno和max_segno之间找一个空闲的段号,即目录中没有这个段号的xlog文件
if ((segno) >= max_segno){
/ Failed to find a free slot within specified range /
if (use_lock)
LWLockRelease(ControlFileLock);
return false;
}
(segno)++;
XLogFilePath(path, ThisTimeLineID, segno);
}
}
if (durable_link_or_rename(tmppath, path, LOG) != 0){//将tmppath重命名为path并持久化
if (use_lock)
LWLockRelease(ControlFileLock);
/ durable_link_or_rename already emitted log message */
return false;
}
if (use_lock)
LWLockRelease(ControlFileLock);
return true;
}

本文转载于:http://win-man.com/
本文关键词:高防CDN    加速CDN    免备案CDN
作者:网站高防专家

发表评论

电子邮件地址不会被公开。 必填项已用*标注