New replica with pgbackrest

I ran the following command on the replica:

(REPLICA) postgres@pgdedicated3:~$ psql --cluster=9.4/main bpd_production

psql: FATAL:  the database system is starting up

-----------------------------------------------------------------------

# (MASTER) postgresql.conf

# WAL/Archive settings

archive_command = 'pgbackrest --stanza=prod-9.4 archive-push %p'

#archive_command = 'true'

checkpoint_segments = 32

checkpoint_timeout = 10min

wal_level = hot_standby

#wal_sender_delay = 1s

max_wal_senders = 5

wal_buffers = 16MB

archive_timeout = 1h

archive_mode = on

checkpoint_completion_target = 0.9 # Spread I/O up to 90% of checkpoint timeout

log_autovacuum_min_duration = 1000      # -1 disables, 0 logs all actions and

shared_preload_libraries = 'pg_stat_statements'

pg_stat_statements.max = 10000

pg_stat_statements.track = all

-----------------------------------------------------------------------

# (REPLICA) postgresql.conf

checkpoint_segments = 32

checkpoint_timeout = 10min

wal_level = hot_standby

max_wal_senders = 5

checkpoint_completion_target = 0.9 # Spread I/O up to 90% of checkpoint timeout

-----------------------------------------------------------------------

# (REPLICA) tail -n 100 -f /var/log/postgresql/postgresql-9.4-main.log

2018-09-23 10:22:14.111 P00   INFO: unable to find 0000000100000CB600000053 in the archive

2018-09-23 10:22:14.111 P00   INFO: got WAL segment 0000000100000CB600000053

2018-09-23 10:22:14.111 P00   INFO: archive-get command end: aborted with exception [001]

2018-09-23 10:22:14.128 CDT [10073-1]: LOG:  started streaming WAL from primary at CB6/53000000 on timeline 1

2018-09-23 10:25:43.081 CDT [10089-1]: [unknown] [unknown]@[local] 00000 [unknown] LOG:  connection received: host=[local]

2018-09-23 10:25:43.082 CDT [10089-2]: bpd_production postgres@[local] 57P03 [unknown] FATAL:  the database system is starting up

2018-09-23 10:25:47.797 CDT [10101-1]: [unknown] [unknown]@[local] 00000 [unknown] LOG:  connection received: host=[local]

2018-09-23 10:25:47.797 CDT [10101-2]: bpd_production postgres@[local] 57P03 [unknown] FATAL:  the database system is starting up

2018-09-23 10:26:11.388 CDT [10114-1]: [unknown] [unknown]@[local] 00000 [unknown] LOG:  connection received: host=[local]

2018-09-23 10:26:11.388 CDT [10114-2]: bpd_production postgres@[local] 57P03 [unknown] FATAL:  the database system is starting up

2018-09-23 10:26:30.021 CDT [10126-1]: [unknown] [unknown]@[local] 00000 [unknown] LOG:  connection received: host=[local]

2018-09-23 10:26:30.021 CDT [10126-2]: bpd_production postgres@[local] 57P03 [unknown] FATAL:  the database system is starting up

2018-09-23 10:27:57.129 CDT [10139-1]: [unknown] [unknown]@[local] 00000 [unknown] LOG:  connection received: host=[local]

2018-09-23 10:27:57.129 CDT [10139-2]: bpd_production postgres@[local] 57P03 [unknown] FATAL:  the database system is starting up

2018-09-23 10:31:02.015 CDT [9912-2]: LOG:  restartpoint complete: wrote 61858 buffers (5.9%); 0 transaction log file(s) added, 0 removed, 0 recycled; write=539.476 s, sync=0.013 s, total=539.490 s; sync files=534, longest=0.000 s, average=0.000 s

2018-09-23 10:31:02.015 CDT [9912-3]: LOG:  recovery restart point at CB6/2E9EDFF0

2018-09-23 10:31:02.015 CDT [9912-4]: DETAIL:  last completed transaction was at log time 2018-09-23 10:30:58.4894-05

2018-09-23 10:31:02.016 CDT [9912-5]: LOG:  restartpoint starting: xlog

-----------------------------------------------------------------------

# (MASTER) $ ls -lah /var/lib/postgresql/9.4/main/pg_xlog/

total 1.1G

drwx------  3 postgres postgres 176K Sep 23 10:34 .

drwx------ 18 postgres postgres 4.0K Sep 23 06:33 ..

-rw-------  1 postgres postgres  341 Sep 23 06:33 0000000100000CB600000019.00000028.backup

-rw-------  1 postgres postgres  16M Sep 23 10:22 0000000100000CB600000053

-rw-------  1 postgres postgres  16M Sep 23 10:27 0000000100000CB600000054

-rw-------  1 postgres postgres  16M Sep 23 10:30 0000000100000CB600000055

-rw-------  1 postgres postgres  16M Sep 23 10:32 0000000100000CB600000056

-rw-------  1 postgres postgres  16M Sep 23 10:37 0000000100000CB600000057

-rw-------  1 postgres postgres  16M Sep 23 00:32 0000000100000CB600000058

-rw-------  1 postgres postgres  16M Sep 23 01:12 0000000100000CB600000059

-rw-------  1 postgres postgres  16M Sep 23 01:30 0000000100000CB60000005A

-rw-------  1 postgres postgres  16M Sep 23 01:34 0000000100000CB60000005B

-rw-------  1 postgres postgres  16M Sep 23 02:00 0000000100000CB60000005C

-rw-------  1 postgres postgres  16M Sep 23 02:35 0000000100000CB60000005D

-rw-------  1 postgres postgres  16M Sep 23 03:24 0000000100000CB60000005E

-rw-------  1 postgres postgres  16M Sep 23 04:23 0000000100000CB60000005F

-rw-------  1 postgres postgres  16M Sep 23 05:05 0000000100000CB600000060

-rw-------  1 postgres postgres  16M Sep 23 06:04 0000000100000CB600000061

-rw-------  1 postgres postgres  16M Sep 23 06:33 0000000100000CB600000062

-rw-------  1 postgres postgres  16M Sep 23 07:00 0000000100000CB600000063

-rw-------  1 postgres postgres  16M Sep 23 07:03 0000000100000CB600000064

-rw-------  1 postgres postgres  16M Sep 23 07:20 0000000100000CB600000065

-rw-------  1 postgres postgres  16M Sep 23 07:32 0000000100000CB600000066

-rw-------  1 postgres postgres  16M Sep 23 07:51 0000000100000CB600000067

-rw-------  1 postgres postgres  16M Sep 23 08:06 0000000100000CB600000068

-rw-------  1 postgres postgres  16M Sep 23 08:00 0000000100000CB600000069

-rw-------  1 postgres postgres  16M Sep 23 08:11 0000000100000CB60000006A

-rw-------  1 postgres postgres  16M Sep 23 08:22 0000000100000CB60000006B

-rw-------  1 postgres postgres  16M Sep 23 08:30 0000000100000CB60000006C

-rw-------  1 postgres postgres  16M Sep 23 08:37 0000000100000CB60000006D

-rw-------  1 postgres postgres  16M Sep 23 08:45 0000000100000CB60000006E

-rw-------  1 postgres postgres  16M Sep 23 08:53 0000000100000CB60000006F

-rw-------  1 postgres postgres  16M Sep 23 09:05 0000000100000CB600000070

-rw-------  1 postgres postgres  16M Sep 23 09:03 0000000100000CB600000071

-rw-------  1 postgres postgres  16M Sep 23 09:01 0000000100000CB600000072

-rw-------  1 postgres postgres  16M Sep 23 09:00 0000000100000CB600000073

-rw-------  1 postgres postgres  16M Sep 23 09:07 0000000100000CB600000074

-rw-------  1 postgres postgres  16M Sep 23 09:06 0000000100000CB600000075

-rw-------  1 postgres postgres  16M Sep 23 09:01 0000000100000CB600000076

-rw-------  1 postgres postgres  16M Sep 23 09:00 0000000100000CB600000077

-rw-------  1 postgres postgres  16M Sep 23 09:04 0000000100000CB600000078

-rw-------  1 postgres postgres  16M Sep 23 09:02 0000000100000CB600000079

-rw-------  1 postgres postgres  16M Sep 23 09:02 0000000100000CB60000007A

-rw-------  1 postgres postgres  16M Sep 23 09:06 0000000100000CB60000007B

-rw-------  1 postgres postgres  16M Sep 23 09:03 0000000100000CB60000007C

-rw-------  1 postgres postgres  16M Sep 23 09:00 0000000100000CB60000007D

-rw-------  1 postgres postgres  16M Sep 23 09:08 0000000100000CB60000007E

-rw-------  1 postgres postgres  16M Sep 23 09:04 0000000100000CB60000007F

-rw-------  1 postgres postgres  16M Sep 23 09:05 0000000100000CB600000080

-rw-------  1 postgres postgres  16M Sep 23 09:12 0000000100000CB600000081

-rw-------  1 postgres postgres  16M Sep 23 09:20 0000000100000CB600000082

-rw-------  1 postgres postgres  16M Sep 23 09:28 0000000100000CB600000083

-rw-------  1 postgres postgres  16M Sep 23 09:32 0000000100000CB600000084

-rw-------  1 postgres postgres  16M Sep 23 09:37 0000000100000CB600000085

-rw-------  1 postgres postgres  16M Sep 23 09:45 0000000100000CB600000086

-rw-------  1 postgres postgres  16M Sep 23 09:58 0000000100000CB600000087

-rw-------  1 postgres postgres  16M Sep 23 09:51 0000000100000CB600000088

-rw-------  1 postgres postgres  16M Sep 23 10:01 0000000100000CB600000089

-rw-------  1 postgres postgres  16M Sep 23 10:02 0000000100000CB60000008A

-rw-------  1 postgres postgres  16M Sep 23 10:01 0000000100000CB60000008B

-rw-------  1 postgres postgres  16M Sep 23 10:03 0000000100000CB60000008C

-rw-------  1 postgres postgres  16M Sep 23 10:02 0000000100000CB60000008D

-rw-------  1 postgres postgres  16M Sep 23 10:01 0000000100000CB60000008E

-rw-------  1 postgres postgres  16M Sep 23 10:00 0000000100000CB60000008F

-rw-------  1 postgres postgres  16M Sep 23 10:03 0000000100000CB600000090

-rw-------  1 postgres postgres  16M Sep 23 10:05 0000000100000CB600000091

-rw-------  1 postgres postgres  16M Sep 23 10:18 0000000100000CB600000092

-rw-------  1 postgres postgres  16M Sep 23 10:15 0000000100000CB600000093

-rw-------  1 postgres postgres  16M Sep 23 10:10 0000000100000CB600000094

drwx------  2 postgres postgres 224K Sep 23 10:34 archive_status

-----------------------------------------------------------------------

# /backup (NFS accessible to both machines)

$ ls -lah /backup/backrest127/archive/prod-9.4/9.4-1

total 6.4M

drwxr-x--- 151 postgres postgres  16K Sep 23 06:42 .

drwxr-x---   3 postgres postgres 4.0K Jan 10  2018 ..

drwxr-x---   2 postgres postgres  40K Sep 23 06:42 0000000100000C22

drwxr-x---   2 postgres postgres  40K Aug  6 12:08 0000000100000C23

drwxr-x---   2 postgres postgres  40K Aug  6 12:52 0000000100000C24

drwxr-x---   2 postgres postgres  40K Aug  6 16:45 0000000100000C25

drwxr-x---   2 postgres postgres  40K Aug  7 10:35 0000000100000C26

drwxr-x---   2 postgres postgres  40K Aug  7 13:38 0000000100000C27

drwxr-x---   2 postgres postgres  40K Aug  7 16:26 0000000100000C28

drwxr-x---   2 postgres postgres  40K Aug  8 09:39 0000000100000C29

drwxr-x---   2 postgres postgres  40K Aug  8 14:19 0000000100000C2A

drwxr-x---   2 postgres postgres  40K Aug  9 09:19 0000000100000C2B

drwxr-x---   2 postgres postgres  40K Aug  9 12:04 0000000100000C2C

drwxr-x---   2 postgres postgres  40K Aug  9 20:04 0000000100000C2D

drwxr-x---   2 postgres postgres  40K Aug 10 10:48 0000000100000C2E

drwxr-x---   2 postgres postgres  40K Aug 10 16:41 0000000100000C2F

drwxr-x---   2 postgres postgres  40K Aug 11 09:11 0000000100000C30

drwxr-x---   2 postgres postgres  40K Aug 12 00:09 0000000100000C31

drwxr-x---   2 postgres postgres  40K Aug 12 00:21 0000000100000C32

drwxr-x---   2 postgres postgres  40K Aug 12 00:29 0000000100000C33

drwxr-x---   2 postgres postgres  40K Aug 12 00:35 0000000100000C34

drwxr-x---   2 postgres postgres  40K Aug 12 14:56 0000000100000C35

drwxr-x---   2 postgres postgres  40K Aug 13 10:43 0000000100000C36

drwxr-x---   2 postgres postgres  40K Aug 13 12:16 0000000100000C37

drwxr-x---   2 postgres postgres  40K Aug 13 17:54 0000000100000C38

drwxr-x---   2 postgres postgres  40K Aug 14 09:08 0000000100000C39

drwxr-x---   2 postgres postgres  40K Aug 14 13:23 0000000100000C3A

drwxr-x---   2 postgres postgres  40K Aug 14 18:16 0000000100000C3B

drwxr-x---   2 postgres postgres  40K Aug 15 12:00 0000000100000C3C

drwxr-x---   2 postgres postgres  40K Aug 15 19:14 0000000100000C3D

drwxr-x---   2 postgres postgres  40K Aug 16 11:30 0000000100000C3E

drwxr-x---   2 postgres postgres  40K Aug 16 17:36 0000000100000C3F

drwxr-x---   2 postgres postgres  40K Aug 16 23:20 0000000100000C40

drwxr-x---   2 postgres postgres  40K Aug 17 14:22 0000000100000C41

drwxr-x---   2 postgres postgres  40K Aug 17 18:55 0000000100000C42

drwxr-x---   2 postgres postgres  40K Aug 19 00:05 0000000100000C43

drwxr-x---   2 postgres postgres  40K Aug 19 00:20 0000000100000C44

drwxr-x---   2 postgres postgres  40K Aug 19 00:28 0000000100000C45

drwxr-x---   2 postgres postgres  40K Aug 19 00:35 0000000100000C46

drwxr-x---   2 postgres postgres  40K Aug 19 11:22 0000000100000C47

drwxr-x---   2 postgres postgres  40K Aug 20 08:01 0000000100000C48

drwxr-x---   2 postgres postgres  40K Aug 20 11:58 0000000100000C49

drwxr-x---   2 postgres postgres  40K Aug 20 13:14 0000000100000C4A

drwxr-x---   2 postgres postgres  40K Aug 20 19:51 0000000100000C4B

drwxr-x---   2 postgres postgres  40K Aug 21 10:59 0000000100000C4C

drwxr-x---   2 postgres postgres  40K Aug 21 16:07 0000000100000C4D

drwxr-x---   2 postgres postgres  40K Aug 21 21:37 0000000100000C4E

drwxr-x---   2 postgres postgres  40K Aug 22 10:31 0000000100000C4F

drwxr-x---   2 postgres postgres  40K Aug 22 12:45 0000000100000C50

drwxr-x---   2 postgres postgres  40K Aug 22 20:04 0000000100000C51

drwxr-x---   2 postgres postgres  40K Aug 23 11:51 0000000100000C52

drwxr-x---   2 postgres postgres  40K Aug 23 17:09 0000000100000C53

drwxr-x---   2 postgres postgres  40K Aug 24 10:08 0000000100000C54

drwxr-x---   2 postgres postgres  40K Aug 24 12:37 0000000100000C55

drwxr-x---   2 postgres postgres  40K Aug 25 09:03 0000000100000C56

drwxr-x---   2 postgres postgres  40K Aug 25 10:36 0000000100000C57

drwxr-x---   2 postgres postgres  40K Aug 26 00:05 0000000100000C58

drwxr-x---   2 postgres postgres  40K Aug 26 00:18 0000000100000C59

drwxr-x---   2 postgres postgres  40K Aug 26 00:26 0000000100000C5A

drwxr-x---   2 postgres postgres  40K Aug 26 00:32 0000000100000C5B

drwxr-x---   2 postgres postgres  40K Aug 26 17:04 0000000100000C5C

drwxr-x---   2 postgres postgres  40K Aug 27 11:05 0000000100000C5D

drwxr-x---   2 postgres postgres  40K Aug 27 13:20 0000000100000C5E

drwxr-x---   2 postgres postgres  40K Aug 27 16:17 0000000100000C5F

drwxr-x---   2 postgres postgres  40K Aug 27 17:32 0000000100000C60

drwxr-x---   2 postgres postgres  40K Aug 28 09:26 0000000100000C61

drwxr-x---   2 postgres postgres  40K Aug 28 15:10 0000000100000C62

drwxr-x---   2 postgres postgres  40K Aug 28 19:07 0000000100000C63

drwxr-x---   2 postgres postgres  40K Aug 29 13:00 0000000100000C64

drwxr-x---   2 postgres postgres  40K Aug 29 18:11 0000000100000C65

drwxr-x---   2 postgres postgres  40K Aug 30 12:24 0000000100000C66

drwxr-x---   2 postgres postgres  40K Aug 30 16:31 0000000100000C67

drwxr-x---   2 postgres postgres  40K Aug 31 11:30 0000000100000C68

drwxr-x---   2 postgres postgres  40K Aug 31 14:32 0000000100000C69

drwxr-x---   2 postgres postgres  40K Sep  1 14:06 0000000100000C6A

drwxr-x---   2 postgres postgres  40K Sep  2 00:09 0000000100000C6B

drwxr-x---   2 postgres postgres  40K Sep  2 00:14 0000000100000C6C

drwxr-x---   2 postgres postgres  40K Sep  2 00:26 0000000100000C6D

drwxr-x---   2 postgres postgres  40K Sep  2 00:33 0000000100000C6E

drwxr-x---   2 postgres postgres  40K Sep  2 00:40 0000000100000C6F

drwxr-x---   2 postgres postgres  40K Sep  2 09:07 0000000100000C70

drwxr-x---   2 postgres postgres  40K Sep  3 11:12 0000000100000C71

drwxr-x---   2 postgres postgres  40K Sep  4 09:04 0000000100000C72

drwxr-x---   2 postgres postgres  40K Sep  4 11:43 0000000100000C73

drwxr-x---   2 postgres postgres  40K Sep  4 17:01 0000000100000C74

drwxr-x---   2 postgres postgres  40K Sep  5 07:54 0000000100000C75

drwxr-x---   2 postgres postgres  40K Sep  5 14:00 0000000100000C76

drwxr-x---   2 postgres postgres  40K Sep  5 17:30 0000000100000C77

drwxr-x---   2 postgres postgres  40K Sep  6 11:50 0000000100000C78

drwxr-x---   2 postgres postgres  40K Sep  6 15:18 0000000100000C79

drwxr-x---   2 postgres postgres  40K Sep  6 23:11 0000000100000C7A

drwxr-x---   2 postgres postgres  40K Sep  7 14:03 0000000100000C7B

drwxr-x---   2 postgres postgres  40K Sep  7 18:05 0000000100000C7C

drwxr-x---   2 postgres postgres  40K Sep  7 21:11 0000000100000C7D

drwxr-x---   2 postgres postgres  40K Sep  7 23:05 0000000100000C7E

drwxr-x---   2 postgres postgres  40K Sep  8 04:35 0000000100000C7F

drwxr-x---   2 postgres postgres  40K Sep  8 10:21 0000000100000C80

drwxr-x---   2 postgres postgres  40K Sep  9 00:05 0000000100000C81

drwxr-x---   2 postgres postgres  40K Sep  9 00:18 0000000100000C82

drwxr-x---   2 postgres postgres  40K Sep  9 00:26 0000000100000C83

drwxr-x---   2 postgres postgres  40K Sep  9 00:33 0000000100000C84

drwxr-x---   2 postgres postgres  40K Sep  9 14:05 0000000100000C85

drwxr-x---   2 postgres postgres  40K Sep 10 10:01 0000000100000C86

drwxr-x---   2 postgres postgres  40K Sep 10 11:50 0000000100000C87

drwxr-x---   2 postgres postgres  40K Sep 10 16:03 0000000100000C88

drwxr-x---   2 postgres postgres  40K Sep 10 22:29 0000000100000C89

drwxr-x---   2 postgres postgres  40K Sep 11 11:48 0000000100000C8A

drwxr-x---   2 postgres postgres  40K Sep 11 16:09 0000000100000C8B

drwxr-x---   2 postgres postgres  40K Sep 12 07:00 0000000100000C8C

drwxr-x---   2 postgres postgres  40K Sep 12 13:31 0000000100000C8D

drwxr-x---   2 postgres postgres  40K Sep 12 15:35 0000000100000C8E

drwxr-x---   2 postgres postgres  40K Sep 13 11:01 0000000100000C8F

drwxr-x---   2 postgres postgres  40K Sep 13 12:53 0000000100000C90

drwxr-x---   2 postgres postgres  40K Sep 13 18:48 0000000100000C91

drwxr-x---   2 postgres postgres  40K Sep 13 20:17 0000000100000C92

drwxr-x---   2 postgres postgres  40K Sep 13 23:29 0000000100000C93

drwxr-x---   2 postgres postgres  40K Sep 14 04:48 0000000100000C94

drwxr-x---   2 postgres postgres  40K Sep 14 11:59 0000000100000C95

drwxr-x---   2 postgres postgres  40K Sep 14 15:34 0000000100000C96

drwxr-x---   2 postgres postgres  40K Sep 15 11:06 0000000100000C97

drwxr-x---   2 postgres postgres  40K Sep 15 12:34 0000000100000C98

drwxr-x---   2 postgres postgres  40K Sep 16 00:13 0000000100000C99

drwxr-x---   2 postgres postgres  40K Sep 16 00:23 0000000100000C9A

drwxr-x---   2 postgres postgres  40K Sep 16 00:31 0000000100000C9B

drwxr-x---   2 postgres postgres  40K Sep 16 09:01 0000000100000C9C

drwxr-x---   2 postgres postgres  40K Sep 17 09:20 0000000100000C9D

drwxr-x---   2 postgres postgres  40K Sep 17 12:07 0000000100000C9E

drwxr-x---   2 postgres postgres  40K Sep 17 12:23 0000000100000C9F

drwxr-x---   2 postgres postgres  40K Sep 17 14:55 0000000100000CA0

drwxr-x---   2 postgres postgres  40K Sep 17 18:49 0000000100000CA1

drwxr-x---   2 postgres postgres  40K Sep 18 01:10 0000000100000CA2

drwxr-x---   2 postgres postgres  40K Sep 18 11:26 0000000100000CA3

drwxr-x---   2 postgres postgres  40K Sep 18 12:40 0000000100000CA4

drwxr-x---   2 postgres postgres  40K Sep 18 17:00 0000000100000CA5

drwxr-x---   2 postgres postgres  40K Sep 19 10:19 0000000100000CA6

drwxr-x---   2 postgres postgres  40K Sep 19 12:31 0000000100000CA7

drwxr-x---   2 postgres postgres  40K Sep 19 20:51 0000000100000CA8

drwxr-x---   2 postgres postgres  40K Sep 20 11:00 0000000100000CA9

drwxr-x---   2 postgres postgres  40K Sep 20 12:22 0000000100000CAA

drwxr-x---   2 postgres postgres  40K Sep 20 15:17 0000000100000CAB

drwxr-x---   2 postgres postgres  40K Sep 20 21:00 0000000100000CAC

drwxr-x---   2 postgres postgres  40K Sep 21 11:41 0000000100000CAD

drwxr-x---   2 postgres postgres  40K Sep 21 14:22 0000000100000CAE

drwxr-x---   2 postgres postgres  40K Sep 21 19:23 0000000100000CAF

drwxr-x---   2 postgres postgres  40K Sep 22 16:58 0000000100000CB0

drwxr-x---   2 postgres postgres  40K Sep 23 00:13 0000000100000CB1

drwxr-x---   2 postgres postgres  40K Sep 23 00:23 0000000100000CB2

drwxr-x---   2 postgres postgres  40K Sep 23 00:32 0000000100000CB3

drwxr-x---   2 postgres postgres  40K Sep 23 00:38 0000000100000CB4

drwxr-x---   2 postgres postgres  40K Sep 23 00:44 0000000100000CB5

drwxr-x---   2 postgres postgres  16K Sep 23 10:32 0000000100000CB6