Pivotal Knowledge Base

Follow

gp_fts_probe_interval / gp_fts_probe_threadcount - Explained

Goal

This document will take a look on how the parameter "gp_fts_probe_interval" work.

Checkout the document / article for more information on how FTS works.  

Solution / Example

On the GPDB master host there is a fault prober process that is forked and monitored from the postmaster postgres process. This fault prober process is also called the FTS (fault tolerance server) process. This process is restarted by the postmaster if it fails.

The FTS is in a continual loop, with sleep between each loop and the time it sleep is determined by the parameter "gp_fts_probe_interval".

And the amount of threads to do the FTS ping is determined by "gp_fts_probe_threadcount" parameter.

Taking a practical example of the same.

-- Lets say our default timeout parameter value is

flightdata=# show gp_fts_probe_interval ;
 gp_fts_probe_interval
-----------------------
 1min
(1 row)

-- And the fts probe process information for my cluster that is running on port 4340 is

[gpadmin@mdw ~]$ ps -ef |grep 4340 | grep fts
gpadmin  21713 21705  0 Feb18 ?        00:00:00 postgres: port  4340, ftsprobe process   con2 con2 con2 con2

-- Strace of the fts process will show initally the process is sleeping like this

[gpadmin@mdw ~]$ strace -p 21713
Process 21713 attached - interrupt to quit
select(0, NULL, NULL, NULL, {6, 71000}) = 

-- After the intreval of "gp_fts_probe_interval" you end up seeing the message stating its has send probes and if the probe was a success then it goes to sleep again.

0 (Timeout)
getppid()                               = 21705
lseek(20, 0, SEEK_END)                  = 32768
lseek(22, 0, SEEK_END)                  = 32768
clone(child_stack=0x2b097a761230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a7619d0, tls=0x2b097a761940, child_tidptr=0x2b097a7619d0) = 431
clone(child_stack=0x2b097a7a2230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a7a29d0, tls=0x2b097a7a2940, child_tidptr=0x2b097a7a29d0) = 432
clone(child_stack=0x2b097a7e3230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a7e39d0, tls=0x2b097a7e3940, child_tidptr=0x2b097a7e39d0) = 433
clone(child_stack=0x2b097a824230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a8249d0, tls=0x2b097a824940, child_tidptr=0x2b097a8249d0) = 434
clone(child_stack=0x2b097a865230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a8659d0, tls=0x2b097a865940, child_tidptr=0x2b097a8659d0) = 435
clone(child_stack=0x2b097a8a6230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a8a69d0, tls=0x2b097a8a6940, child_tidptr=0x2b097a8a69d0) = 436
clone(child_stack=0x2b097a8e7230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a8e79d0, tls=0x2b097a8e7940, child_tidptr=0x2b097a8e79d0) = 437
clone(child_stack=0x2b097a928230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a9289d0, tls=0x2b097a928940, child_tidptr=0x2b097a9289d0) = 438
clone(child_stack=0x2b097a969230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a9699d0, tls=0x2b097a969940, child_tidptr=0x2b097a9699d0) = 439
clone(child_stack=0x2b097a9aa230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a9aa9d0, tls=0x2b097a9aa940, child_tidptr=0x2b097a9aa9d0) = 440
clone(child_stack=0x2b097a9eb230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097a9eb9d0, tls=0x2b097a9eb940, child_tidptr=0x2b097a9eb9d0) = 441
clone(child_stack=0x2b097aa2c230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097aa2c9d0, tls=0x2b097aa2c940, child_tidptr=0x2b097aa2c9d0) = 442
clone(child_stack=0x2b097aa6d230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097aa6d9d0, tls=0x2b097aa6d940, child_tidptr=0x2b097aa6d9d0) = 443
clone(child_stack=0x2b097aaae230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097aaae9d0, tls=0x2b097aaae940, child_tidptr=0x2b097aaae9d0) = 444
clone(child_stack=0x2b097aaef230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097aaef9d0, tls=0x2b097aaef940, child_tidptr=0x2b097aaef9d0) = 445
clone(child_stack=0x2b097ab30230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x2b097ab309d0, tls=0x2b097ab30940, child_tidptr=0x2b097ab309d0) = 446
futex(0x2b097a7619d0, FUTEX_WAIT, 431, NULL) = 0
futex(0x2b097a8249d0, FUTEX_WAIT, 434, NULL) = 0
select(0, NULL, NULL, NULL, {60, 0})    

-- From the above strace output, the number of clone in the output refers to the total number of child process which is equal to the number of thread defined by "gp_fts_probe_threadcount"

-- If the fts doesn't shows that last line after the fts probe ( i.e select(0, NULL..... )), then there are someone in the cluster who hasn't responded back and its waiting on timeout which can be either from gp_segment_connect_timeout or gp_fts_probe_timeout ping.

Comments

Powered by Zendesk