Redis Sentinel：故障检测、自动切主与参数调优

sentinel monitor mymaster redis-1 6379 2
sentinel down-after-milliseconds mymaster 5000
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 30000

/* 摘自 src/sentinel.c:5105 附近，省略无关代码；中文注释为本文补充。 */
mstime_t master_down_time = 0;
if (master->flags & SRI_S_DOWN)
    master_down_time = mstime() - master->s_down_since_time;

/* Replica 最长可断连时间：
 * Master 已处于 SDOWN 的时长 + 10 * down-after-milliseconds。 */
mstime_t max_master_down_time =
    master_down_time + master->down_after_period * 10;

if (slave->flags & (SRI_S_DOWN|SRI_O_DOWN)) continue; /* 已被判定下线 */
if (slave->link->disconnected) continue;              /* Sentinel 连接已断开 */
if (mstime() - slave->link->last_avail_time >
    sentinel_ping_period * 5) continue;                /* 默认约 5 秒不可用 */
if (slave->slave_priority == 0) continue;              /* 明确禁止晋升 */

/* Master 已进入 SDOWN 时，只接受默认 5 秒内的 INFO；
 * 其他情况下接受默认 30 秒内的 INFO。 */
if (master->flags & SRI_S_DOWN)
    info_validity_time = sentinel_ping_period * 5;
else
    info_validity_time = sentinel_info_period * 3;

if (mstime() - slave->info_refresh > info_validity_time) continue;
if (slave->master_link_down_time > max_master_down_time) continue;

/* 摘自 src/sentinel.c:5077 附近；返回值越小，排序越靠前。 */
if ((*sa)->slave_priority != (*sb)->slave_priority)
    return (*sa)->slave_priority - (*sb)->slave_priority; /* priority 小者优先 */

if ((*sa)->slave_repl_offset > (*sb)->slave_repl_offset)
    return -1; /* offset 大者优先 */
else if ((*sa)->slave_repl_offset < (*sb)->slave_repl_offset)
    return 1;

/* priority 和 offset 都相同时，没有 run ID 的节点排在最后。 */
if (sa_runid == NULL && sb_runid == NULL) return 0;
else if (sa_runid == NULL) return 1;
else if (sb_runid == NULL) return -1;

return strcasecmp(sa_runid, sb_runid); /* run ID 字典序小者优先 */

sentinel_ip,sentinel_port,sentinel_runid,current_epoch,
master_name,master_ip,master_port,master_config_epoch

for sentinel in sentinel-1 sentinel-2 sentinel-3; do
  printf '\n# %s\n' "$sentinel"
  printf 'myid='
  pc exec -T "$sentinel" redis-cli --raw -p 26379 \
    SENTINEL myid
  printf 'master='
  pc exec -T "$sentinel" redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | paste -sd ':' -
  printf 'ckquorum='
  pc exec -T "$sentinel" redis-cli --raw -p 26379 \
    SENTINEL ckquorum mymaster
  pc exec -T "$sentinel" redis-cli --raw -p 26379 \
    SENTINEL replicas mymaster | tr -d '\r' | \
    awk '$0 == "name" { replicas++ } END { print "replicas=" replicas + 0 }'
  pc exec -T "$sentinel" redis-cli --raw -p 26379 \
    SENTINEL sentinels mymaster | tr -d '\r' | \
    awk '$0 == "name" { peers++ } END { print "peer-sentinels=" peers + 0 }'
done

pc exec -T sentinel-1 redis-cli --raw -p 26379 \
  SENTINEL sentinels mymaster | tr -d '\r' | \
  awk '
    $0 == "name" {
      peer++
      if (peer > 1) print ""
      printf "# peer %d\n", peer
    }
    { print }
  '

printf '# redis-1 MONITOR\n'
pc exec redis-1 redis-cli MONITOR

FAILED_MASTER=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
  SENTINEL get-master-addr-by-name mymaster | sed -n '1p')
printf '# 停止当前 Master\n'
printf 'old master: %s\n' "$FAILED_MASTER"
pc stop "$FAILED_MASTER"

printf '\n# 轮询三个 Sentinel 的 Master 视图\n'
NEW_MASTER=''
for round in $(seq 0 45); do
  VIEW_1=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster 2>/dev/null | sed -n '1p')
  VIEW_2=$(pc exec -T sentinel-2 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster 2>/dev/null | sed -n '1p')
  VIEW_3=$(pc exec -T sentinel-3 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster 2>/dev/null | sed -n '1p')
  printf '[round %02d] sentinel-1=%s sentinel-2=%s sentinel-3=%s\n' \
    "$round" "$VIEW_1" "$VIEW_2" "$VIEW_3"

  if [ -n "$VIEW_1" ] && \
     [ "$VIEW_1" = "$VIEW_2" ] && \
     [ "$VIEW_2" = "$VIEW_3" ] && \
     [ "$VIEW_1" != "$FAILED_MASTER" ]; then
    NEW_MASTER=$VIEW_1
    break
  fi
  sleep 1
done

test -n "$NEW_MASTER"
printf 'new master: %s\n' "$NEW_MASTER"

# 停止当前 Master
old master: redis-1
redis-sentinel-lab_redis-1_1

# 轮询三个 Sentinel 的 Master 视图
[round 00] sentinel-1=redis-1 sentinel-2=redis-1 sentinel-3=redis-1
[round 01] sentinel-1=redis-1 sentinel-2=redis-1 sentinel-3=redis-1
[round 02] sentinel-1=redis-1 sentinel-2=redis-1 sentinel-3=redis-1
[round 03] sentinel-1=redis-1 sentinel-2=redis-1 sentinel-3=redis-1
[round 04] sentinel-1=redis-2 sentinel-2=redis-2 sentinel-3=redis-2
new master: redis-2

for sentinel in sentinel-1 sentinel-2 sentinel-3; do
  printf '\n# %s\n' "$sentinel"
  pc logs "$sentinel" 2>&1 | \
    grep -E '\+(sdown|odown|new-epoch|try-failover|vote-for-leader|elected-leader|selected-slave|promoted-slave|switch-master)'
done

printf '# 启动旧 Master\n'
pc start "$FAILED_MASTER"

printf '\n# 等待旧 Master 恢复为 Replica\n'
for attempt in $(seq 1 60); do
  if pc exec -T "$FAILED_MASTER" redis-cli --raw INFO replication \
    2>/dev/null | tr -d '\r' | \
    awk -F: '
      $1 == "role" { role=$2 }
      $1 == "master_link_status" { link=$2 }
      END { exit !(role == "slave" && link == "up") }
    '
  then
    printf 'recovered after %ss\n' "$attempt"
    break
  fi
  sleep 1
done

printf '\n# 旧 Master 的复制状态\n'
pc exec -T "$FAILED_MASTER" redis-cli --raw INFO replication | \
  sed -n '/^role:/p;/^master_host:/p;/^master_link_status:/p'

printf '\n# 三个 Sentinel 的 Master 视图\n'
for sentinel in sentinel-1 sentinel-2 sentinel-3; do
  printf '%s -> ' "$sentinel"
  pc exec -T "$sentinel" redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | paste -sd ':' -
done

# 启动旧 Master
redis-sentinel-lab_redis-1_1

# 等待旧 Master 恢复为 Replica
recovered after 33s

# 旧 Master 的复制状态
role:slave
master_host:redis-2
master_link_status:up

# 三个 Sentinel 的 Master 视图
sentinel-1 -> redis-2:6379
sentinel-2 -> redis-2:6379
sentinel-3 -> redis-2:6379

RECOVERING_NODE=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
  SENTINEL get-master-addr-by-name mymaster | sed -n '1p')
printf '# 停止当前 Master: %s\n' "$RECOVERING_NODE"
pc stop "$RECOVERING_NODE"

printf '\n# 等待三个 Sentinel 的 Master 视图收敛\n'
for attempt in $(seq 0 45); do
  MASTER_1=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | paste -sd ':' -)
  MASTER_2=$(pc exec -T sentinel-2 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | paste -sd ':' -)
  MASTER_3=$(pc exec -T sentinel-3 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | paste -sd ':' -)
  if [ "$MASTER_1" = "$MASTER_2" ] && \
     [ "$MASTER_2" = "$MASTER_3" ] && \
     [ "$MASTER_1" != "$RECOVERING_NODE:6379" ]; then
    printf 'consensus reached at round %s\n' "$attempt"
    break
  fi
  sleep 1
done

NEW_MASTER=${MASTER_1%:6379}
printf 'new master: %s\n' "$NEW_MASTER"

# 停止当前 Master: redis-1
redis-sentinel-lab_redis-1_1

# 等待三个 Sentinel 的 Master 视图收敛
consensus reached at round 4
new master: redis-3

printf '# 写入实验键\n'
pc exec -T sentinel-1 redis-cli -h "$NEW_MASTER" \
  SET lab:recovery-window ready
printf '\n# 写入约 10 MB 数据\n'
pc exec -T sentinel-1 redis-benchmark -h "$NEW_MASTER" \
  -t set -n 10000 -d 1024 -r 1000000 -q

# 写入实验键
OK

# 写入约 10 MB 数据
SET: 175438.59 requests per second, p50=0.207 msec

RECOVERING_NODE=redis-1
NEW_MASTER=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
  SENTINEL get-master-addr-by-name mymaster | sed -n '1p')

while :; do
  date '+%H:%M:%S'
  pc exec -T sentinel-1 redis-cli --raw -p 26379 \
    SENTINEL replicas mymaster 2>/dev/null | tr -d '\r' | \
    awk -v target="$RECOVERING_NODE" '
      function emit() {
        if (name ~ ("^" target ":") || ip == target) {
          printf "name=%s ip=%s flags=%s link=%s link-down-ms=%s\n", \
            name, ip, flags, link, down
        }
      }
      $0 == "name" {
        emit()
        name=""; ip=""; flags=""; link=""; down=""
        getline; name=$0; next
      }
      $0 == "ip" { getline; ip=$0; next }
      $0 == "flags" { getline; flags=$0; next }
      $0 == "master-link-status" { getline; link=$0; next }
      $0 == "master-link-down-time" { getline; down=$0; next }
      END { emit() }
    '

  pc exec -T "$RECOVERING_NODE" redis-cli --raw INFO replication \
    2>/dev/null | \
    sed -n '/^role:/p;/^master_host:/p;/^master_link_status:/p;/^master_sync_in_progress:/p'
  sleep 1
done

pc start redis-1

# 恢复节点状态转换
[00s] sentinel: flags=slave,disconnected link=err link-down-ms=0 | redis: role=master
[01s] sentinel: flags=slave link=err link-down-ms=0 | redis: role=master
[11s] sentinel: flags=slave link=err link-down-ms=-1000 | redis: role=slave master=redis-3 link=down sync=0
[42s] sentinel: flags=slave link=ok link-down-ms=0 | redis: role=slave master=redis-3 link=up sync=0

printf '# 从恢复后的 Replica 读取实验键\n'
pc exec -T "$RECOVERING_NODE" redis-cli GET lab:recovery-window
printf '\n# 新 Master 的同步日志\n'
pc logs "$NEW_MASTER" 2>&1 | \
  grep -E 'Full resync|BGSAVE|diskless|Synchronization with replica'

# 从恢复后的 Replica 读取实验键
ready

# 新 Master 的同步日志
1:M 04 Aug 2026 08:36:01.434 * Full resync requested by replica redis-1:6379 (rdb-channel)
1:M 04 Aug 2026 08:36:01.434 * Delay next BGSAVE for diskless SYNC
1:M 04 Aug 2026 08:36:31.791 * Starting BGSAVE for SYNC with target: replicas sockets (rdb-channel)
54:C 04 Aug 2026 08:36:31.835 * BGSAVE done, 9962 keys saved, 0 keys skipped, 10399539 bytes written.
1:M 04 Aug 2026 08:36:31.859 * Synchronization with replica redis-1:6379 succeeded

printf '# 检查 Sentinel 可用性\n'
pc exec -T sentinel-1 redis-cli -p 26379 SENTINEL CKQUORUM mymaster

printf '\n# 查询当前 Master\n'
OLD_MASTER=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
  SENTINEL get-master-addr-by-name mymaster | sed -n '1p')
printf 'old master: %s\n' "$OLD_MASTER"

for node in redis-1 redis-2 redis-3; do
  printf '\n# %s: INFO replication\n' "$node"
  pc exec -T "$node" redis-cli --raw INFO replication | \
    sed -n '/^role:/p;/^master_host:/p;/^master_link_status:/p;/^master_repl_offset:/p;/^slave_repl_offset:/p'
done

printf '# 发起计划切换\n'
pc exec -T sentinel-1 redis-cli -p 26379 SENTINEL FAILOVER mymaster

printf '\n# 等待三个 Sentinel 的 Master 视图收敛\n'
for round in $(seq 0 45); do
  MASTER_1=$(pc exec -T sentinel-1 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | sed -n '1p')
  MASTER_2=$(pc exec -T sentinel-2 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | sed -n '1p')
  MASTER_3=$(pc exec -T sentinel-3 redis-cli --raw -p 26379 \
    SENTINEL get-master-addr-by-name mymaster | sed -n '1p')
  printf '[round %02d] sentinel-1=%s sentinel-2=%s sentinel-3=%s\n' \
    "$round" "$MASTER_1" "$MASTER_2" "$MASTER_3"
  if [ "$MASTER_1" = "$MASTER_2" ] && \
     [ "$MASTER_2" = "$MASTER_3" ] && \
     [ "$MASTER_1" != "$OLD_MASTER" ]; then
    NEW_MASTER=$MASTER_1
    break
  fi
  sleep 1
done

printf 'master moved: %s -> %s\n' "$OLD_MASTER" "$NEW_MASTER"
printf '\n# 验证新 Master 读写\n'
pc exec -T sentinel-1 redis-cli -h "$NEW_MASTER" \
  SET lab:planned-failover ok
pc exec -T sentinel-1 redis-cli -h "$NEW_MASTER" \
  GET lab:planned-failover

# 发起计划切换
OK

# 等待三个 Sentinel 的 Master 视图收敛
[round 00] sentinel-1=redis-1 sentinel-2=redis-1 sentinel-3=redis-1
[round 01] sentinel-1=redis-3 sentinel-2=redis-3 sentinel-3=redis-3
master moved: redis-1 -> redis-3

# 验证新 Master 读写
OK
ok

printf '# 等待旧 Master 恢复为 Replica\n'
for attempt in $(seq 1 60); do
  if pc exec -T "$OLD_MASTER" redis-cli --raw INFO replication \
    2>/dev/null | tr -d '\r' | \
    awk -F: '
      $1 == "role" { role=$2 }
      $1 == "master_link_status" { link=$2 }
      END { exit !(role == "slave" && link == "up") }
    '
  then
    printf 'old master recovered after %ss\n' "$attempt"
    break
  fi
  sleep 1
done

printf '\n# 旧 Master 的复制状态\n'
pc exec -T "$OLD_MASTER" redis-cli --raw INFO replication | \
  sed -n '/^role:/p;/^master_host:/p;/^master_link_status:/p'

printf '\n# 三个 Redis 的最终角色\n'
for node in redis-1 redis-2 redis-3; do
  printf '%s: ' "$node"
  pc exec -T "$node" redis-cli --raw ROLE | sed -n '1p'
done

printf '\n# Sentinel 故障转移事件\n'
for sentinel in sentinel-1 sentinel-2 sentinel-3; do
  printf '\n# %s\n' "$sentinel"
  pc logs "$sentinel" 2>&1 | \
    grep -E '\+(sdown|odown|new-epoch|try-failover|elected-leader|selected-slave|promoted-slave|switch-master)'
done