head	1.72;
access;
symbols
	REL9_0_0:1.68.2.4
	REL9_1_ALPHA1:1.72
	REL9_0_RC1:1.68.2.4
	REL9_0_BETA4:1.68.2.3
	REL9_0_STABLE:1.68.0.2
	REL9_0_BETA3:1.68
	REL9_0_BETA2:1.66
	REL7_4_29:1.25
	REL8_0_25:1.31
	REL8_1_21:1.38
	REL8_2_17:1.42
	REL8_3_11:1.52
	REL8_4_4:1.59
	REL9_0_BETA1:1.66
	REL9_0_ALPHA5_BRANCH:1.65.0.2
	REL9_0_ALPHA5:1.65
	REL7_4_28:1.25
	REL8_0_24:1.31
	REL8_1_20:1.38
	REL8_2_16:1.42
	REL8_3_10:1.52
	REL8_4_3:1.59
	REL9_0_ALPHA4:1.61
	REL9_0_ALPHA4_BRANCH:1.61.0.2
	REL8_5_ALPHA3:1.60
	REL8_5_ALPHA3_BRANCH:1.60.0.2
	REL7_4_27:1.25
	REL8_0_23:1.31
	REL8_1_19:1.38
	REL8_2_15:1.42
	REL8_3_9:1.52
	REL8_4_2:1.59
	REL8_5_ALPHA2:1.59
	REL8_5_ALPHA2_BRANCH:1.59.0.6
	REL7_4_26:1.25
	REL8_0_22:1.31
	REL8_1_18:1.38
	REL8_2_14:1.42
	REL8_3_8:1.52
	REL8_4_1:1.59
	REL8_5_ALPHA1:1.59
	REL8_5_ALPHA1_BRANCH:1.59.0.4
	REL8_4_STABLE:1.59.0.2
	REL8_4_0:1.59
	REL8_4_RC2:1.59
	REL8_4_RC1:1.59
	REL8_4_BETA2:1.59
	REL8_4_BETA1:1.59
	REL7_4_25:1.25
	REL8_0_21:1.31
	REL8_1_17:1.38
	REL8_2_13:1.42
	REL8_3_7:1.52
	REL7_4_24:1.25
	REL8_0_20:1.31
	REL8_1_16:1.38
	REL8_2_12:1.42
	REL8_3_6:1.52
	REL7_4_23:1.25
	REL8_0_19:1.31
	REL8_1_15:1.38
	REL8_2_11:1.42
	REL8_3_5:1.52
	REL7_4_22:1.25
	REL8_0_18:1.31
	REL8_1_14:1.38
	REL8_2_10:1.42
	REL8_3_4:1.52
	REL7_4_21:1.25
	REL8_0_17:1.31
	REL8_1_13:1.38
	REL8_2_9:1.42
	REL8_3_3:1.52
	REL7_4_20:1.25
	REL8_0_16:1.31
	REL8_1_12:1.38
	REL8_2_8:1.42
	REL8_3_2:1.52
	REL8_2_7:1.42
	REL8_3_1:1.52
	REL8_3_STABLE:1.52.0.2
	REL8_3_0:1.52
	REL8_3_RC2:1.52
	REL7_3_21:1.21
	REL7_4_19:1.25
	REL8_0_15:1.31
	REL8_1_11:1.38
	REL8_2_6:1.42
	REL8_3_RC1:1.52
	REL8_3_BETA4:1.46
	REL8_3_BETA3:1.45
	REL8_3_BETA2:1.45
	REL8_3_BETA1:1.45
	REL7_3_20:1.21
	REL7_4_18:1.25
	REL8_0_14:1.31
	REL8_1_10:1.38
	REL8_2_5:1.42
	REL7_3_19:1.21
	REL7_4_17:1.25
	REL8_0_13:1.31
	REL8_1_9:1.38
	REL8_2_4:1.42
	REL8_0_12:1.31
	REL8_1_8:1.38
	REL8_2_3:1.42
	REL7_3_18:1.21
	REL7_4_16:1.25
	REL8_0_11:1.31
	REL8_1_7:1.38
	REL8_2_2:1.42
	REL8_0_10:1.31
	REL8_1_6:1.38
	REL8_2_1:1.42
	REL7_4_15:1.25
	REL7_3_17:1.21
	REL8_2_STABLE:1.42.0.2
	REL8_2_0:1.42
	REL8_2_RC1:1.41
	REL8_2_BETA3:1.41
	REL8_2_BETA2:1.41
	REL8_1_5:1.38
	REL8_0_9:1.31
	REL7_4_14:1.25
	REL7_3_16:1.21
	REL8_2_BETA1:1.40
	REL7_3_15:1.21
	REL7_4_13:1.25
	REL8_0_8:1.31
	REL8_1_4:1.38
	REL7_3_14:1.21
	REL7_4_12:1.25
	REL8_0_7:1.31
	REL8_1_3:1.38
	REL7_3_13:1.21
	REL7_4_11:1.25
	REL8_0_6:1.31
	REL8_1_2:1.38
	REL7_3_12:1.21
	REL7_4_10:1.25
	REL8_0_5:1.31
	REL8_1_1:1.38
	REL8_1_STABLE:1.38.0.2
	REL8_1_0:1.38
	REL8_1_0RC1:1.37
	REL8_1_0BETA4:1.37
	REL8_1_0BETA3:1.35
	REL7_3_11:1.21
	REL7_4_9:1.25
	REL8_0_4:1.31
	REL8_1_0BETA2:1.31
	REL8_1_0BETA1:1.31
	REL7_2_8:1.15
	REL7_3_10:1.21
	REL7_4_8:1.25
	REL8_0_3:1.31
	REL8_0_2:1.31
	REL7_2_7:1.15
	REL7_3_9:1.21
	REL7_4_7:1.25
	REL8_0_1:1.31
	REL8_0_STABLE:1.31.0.4
	REL8_0_0:1.31.0.2
	REL8_0_0RC5:1.31
	REL8_0_0RC4:1.31
	REL8_0_0RC3:1.31
	REL8_0_0RC2:1.31
	REL8_0_0RC1:1.31
	REL8_0_0BETA5:1.31
	REL8_0_0BETA4:1.29
	REL7_4_6:1.25
	REL7_3_8:1.21
	REL7_2_6:1.15
	REL8_0_0BETA3:1.29
	REL8_0_0BETA2:1.29
	REL7_2_5:1.15
	REL7_4_5:1.25
	REL7_3_7:1.21
	REL7_4_4:1.25
	REL8_0_0BETA1:1.29
	REL7_4_3:1.25
	REL7_4_2:1.25
	REL7_3_6:1.21
	REL7_4_1:1.25
	REL7_3_5:1.21
	REL7_4:1.25
	REL7_4_RC2:1.25
	REL7_4_STABLE:1.25.0.2
	REL7_4_RC1:1.25
	REL7_4_BETA5:1.25
	REL7_4_BETA4:1.25
	REL7_4_BETA3:1.24
	REL7_4_BETA2:1.23
	WIN32_DEV:1.23.0.2
	REL7_4_BETA1:1.23
	REL7_3_4:1.21
	REL7_3_2:1.21
	REL7_2_4:1.15
	REL7_3_STABLE:1.21.0.2
	REL7_2_3:1.15
	REL7_2_STABLE:1.15.0.2
	REL7_2:1.14
	REL7_2_RC2:1.14
	REL7_2_RC1:1.14
	REL7_2_BETA5:1.14
	REL7_2_BETA4:1.14
	REL7_2_BETA3:1.12
	REL7_2_BETA2:1.12
	REL7_2_BETA1:1.11
	REL7_1_2:1.5
	REL7_1_STABLE:1.5.0.2
	REL7_1:1.5;
locks; strict;
comment	@# @;


1.72
date	2010.08.17.04.37.21;	author petere;	state Exp;
branches;
next	1.71;

1.71
date	2010.07.27.19.01.16;	author petere;	state Exp;
branches;
next	1.70;

1.70
date	2010.07.16.11.35.40;	author heikki;	state Exp;
branches;
next	1.69;

1.69
date	2010.07.16.11.20.23;	author heikki;	state Exp;
branches;
next	1.68;

1.68
date	2010.07.08.16.44.12;	author momjian;	state Exp;
branches
	1.68.2.1;
next	1.67;

1.67
date	2010.07.07.14.42.09;	author momjian;	state Exp;
branches;
next	1.66;

1.66
date	2010.04.13.14.15.25;	author momjian;	state Exp;
branches;
next	1.65;

1.65
date	2010.03.17.17.12.31;	author petere;	state Exp;
branches;
next	1.64;

1.64
date	2010.02.27.20.16.17;	author momjian;	state Exp;
branches;
next	1.63;

1.63
date	2010.02.27.01.39.46;	author momjian;	state Exp;
branches;
next	1.62;

1.62
date	2010.02.20.18.28.37;	author momjian;	state Exp;
branches;
next	1.61;

1.61
date	2010.02.03.17.25.06;	author momjian;	state Exp;
branches;
next	1.60;

1.60
date	2009.11.28.16.21.31;	author momjian;	state Exp;
branches;
next	1.59;

1.59
date	2009.04.09.16.20.50;	author momjian;	state Exp;
branches;
next	1.58;

1.58
date	2009.01.15.00.34.25;	author momjian;	state Exp;
branches;
next	1.57;

1.57
date	2008.12.18.22.34.21;	author momjian;	state Exp;
branches;
next	1.56;

1.56
date	2008.12.18.22.21.16;	author momjian;	state Exp;
branches;
next	1.55;

1.55
date	2008.12.10.11.05.49;	author momjian;	state Exp;
branches;
next	1.54;

1.54
date	2008.12.06.21.34.27;	author momjian;	state Exp;
branches;
next	1.53;

1.53
date	2008.05.02.19.52.37;	author tgl;	state Exp;
branches;
next	1.52;

1.52
date	2007.12.29.17.55.07;	author momjian;	state Exp;
branches;
next	1.51;

1.51
date	2007.12.16.14.05.12;	author momjian;	state Exp;
branches;
next	1.50;

1.50
date	2007.12.16.14.03.32;	author momjian;	state Exp;
branches;
next	1.49;

1.49
date	2007.12.10.22.08.36;	author momjian;	state Exp;
branches;
next	1.48;

1.48
date	2007.12.10.14.51.10;	author momjian;	state Exp;
branches;
next	1.47;

1.47
date	2007.12.10.14.05.05;	author momjian;	state Exp;
branches;
next	1.46;

1.46
date	2007.11.28.15.42.31;	author petere;	state Exp;
branches;
next	1.45;

1.45
date	2007.08.01.22.45.07;	author tgl;	state Exp;
branches;
next	1.44;

1.44
date	2007.06.28.00.02.37;	author tgl;	state Exp;
branches;
next	1.43;

1.43
date	2007.01.31.20.56.19;	author momjian;	state Exp;
branches;
next	1.42;

1.42
date	2006.11.25.22.44.48;	author petere;	state Exp;
branches;
next	1.41;

1.41
date	2006.10.23.18.10.32;	author petere;	state Exp;
branches;
next	1.40;

1.40
date	2006.09.16.00.30.16;	author momjian;	state Exp;
branches;
next	1.39;

1.39
date	2006.03.03.22.02.08;	author momjian;	state Exp;
branches;
next	1.38;

1.38
date	2005.11.04.23.14.02;	author petere;	state Exp;
branches;
next	1.37;

1.37
date	2005.10.22.21.56.07;	author tgl;	state Exp;
branches;
next	1.36;

1.36
date	2005.10.13.17.32.42;	author momjian;	state Exp;
branches;
next	1.35;

1.35
date	2005.10.01.01.42.43;	author momjian;	state Exp;
branches;
next	1.34;

1.34
date	2005.09.29.02.29.48;	author momjian;	state Exp;
branches;
next	1.33;

1.33
date	2005.09.28.18.40.28;	author momjian;	state Exp;
branches;
next	1.32;

1.32
date	2005.09.28.18.18.02;	author momjian;	state Exp;
branches;
next	1.31;

1.31
date	2004.11.15.06.32.14;	author neilc;	state Exp;
branches;
next	1.30;

1.30
date	2004.11.15.04.35.57;	author neilc;	state Exp;
branches;
next	1.29;

1.29
date	2004.08.08.04.34.43;	author tgl;	state Exp;
branches;
next	1.28;

1.28
date	2004.03.09.16.57.47;	author neilc;	state Exp;
branches;
next	1.27;

1.27
date	2004.01.06.17.26.23;	author neilc;	state Exp;
branches;
next	1.26;

1.26
date	2003.11.29.19.51.38;	author pgsql;	state Exp;
branches;
next	1.25;

1.25
date	2003.09.20.20.12.05;	author tgl;	state Exp;
branches;
next	1.24;

1.24
date	2003.08.31.17.32.20;	author petere;	state Exp;
branches;
next	1.23;

1.23
date	2003.03.24.14.32.51;	author petere;	state Exp;
branches
	1.23.2.1;
next	1.22;

1.22
date	2002.11.15.02.44.54;	author momjian;	state Exp;
branches;
next	1.21;

1.21
date	2002.11.02.22.23.01;	author tgl;	state Exp;
branches;
next	1.20;

1.20
date	2002.09.06.20.26.00;	author momjian;	state Exp;
branches;
next	1.19;

1.19
date	2002.09.06.20.08.06;	author momjian;	state Exp;
branches;
next	1.18;

1.18
date	2002.08.30.16.53.34;	author momjian;	state Exp;
branches;
next	1.17;

1.17
date	2002.08.30.16.50.50;	author momjian;	state Exp;
branches;
next	1.16;

1.16
date	2002.07.05.19.06.11;	author momjian;	state Exp;
branches;
next	1.15;

1.15
date	2002.02.11.23.25.14;	author momjian;	state Exp;
branches;
next	1.14;

1.14
date	2001.11.28.20.49.10;	author petere;	state Exp;
branches;
next	1.13;

1.13
date	2001.11.21.06.09.45;	author thomas;	state Exp;
branches;
next	1.12;

1.12
date	2001.10.26.23.10.21;	author tgl;	state Exp;
branches;
next	1.11;

1.11
date	2001.09.29.04.02.19;	author tgl;	state Exp;
branches;
next	1.10;

1.10
date	2001.09.22.03.59.17;	author momjian;	state Exp;
branches;
next	1.9;

1.9
date	2001.09.09.23.52.12;	author petere;	state Exp;
branches;
next	1.8;

1.8
date	2001.08.25.18.52.41;	author tgl;	state Exp;
branches;
next	1.7;

1.7
date	2001.05.17.21.50.16;	author petere;	state Exp;
branches;
next	1.6;

1.6
date	2001.05.15.13.57.37;	author momjian;	state Exp;
branches;
next	1.5;

1.5
date	2001.03.16.05.44.33;	author tgl;	state Exp;
branches;
next	1.4;

1.4
date	2001.03.13.01.17.05;	author tgl;	state Exp;
branches;
next	1.3;

1.3
date	2001.02.26.00.50.07;	author tgl;	state Exp;
branches;
next	1.2;

1.2
date	2001.02.18.04.50.43;	author tgl;	state Exp;
branches;
next	1.1;

1.1
date	2001.01.24.23.15.19;	author petere;	state Exp;
branches;
next	;

1.23.2.1
date	2003.09.07.04.36.44;	author momjian;	state Exp;
branches;
next	;

1.68.2.1
date	2010.07.16.11.21.46;	author heikki;	state Exp;
branches;
next	1.68.2.2;

1.68.2.2
date	2010.07.16.11.36.10;	author heikki;	state Exp;
branches;
next	1.68.2.3;

1.68.2.3
date	2010.07.27.19.01.04;	author petere;	state Exp;
branches;
next	1.68.2.4;

1.68.2.4
date	2010.08.17.04.37.19;	author petere;	state Exp;
branches;
next	;


desc
@@


1.72
log
@Spell and markup checking
@
text
@<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.71 2010/07/27 19:01:16 petere Exp $ -->

<chapter id="wal">
 <title>Reliability and the Write-Ahead Log</title>

 <para>
  This chapter explains how the Write-Ahead Log is used to obtain
  efficient, reliable operation.
 </para>

 <sect1 id="wal-reliability">
  <title>Reliability</title>

  <para>
   Reliability is an important property of any serious database
   system, and <productname>PostgreSQL</> does everything possible to
   guarantee reliable operation. One aspect of reliable operation is
   that all data recorded by a committed transaction should be stored
   in a nonvolatile area that is safe from power loss, operating
   system failure, and hardware failure (except failure of the
   nonvolatile area itself, of course).  Successfully writing the data
   to the computer's permanent storage (disk drive or equivalent)
   ordinarily meets this requirement.  In fact, even if a computer is
   fatally damaged, if the disk drives survive they can be moved to
   another computer with similar hardware and all committed
   transactions will remain intact.
  </para>

  <para>
   While forcing data periodically to the disk platters might seem like
   a simple operation, it is not. Because disk drives are dramatically
   slower than main memory and CPUs, several layers of caching exist
   between the computer's main memory and the disk platters.
   First, there is the operating system's buffer cache, which caches
   frequently requested disk blocks and combines disk writes. Fortunately,
   all operating systems give applications a way to force writes from
   the buffer cache to disk, and <productname>PostgreSQL</> uses those
   features.  (See the <xref linkend="guc-wal-sync-method"> parameter
   to adjust how this is done.)
  </para>

  <para>
   Next, there might be a cache in the disk drive controller; this is
   particularly common on <acronym>RAID</> controller cards. Some of
   these caches are <firstterm>write-through</>, meaning writes are sent
   to the drive as soon as they arrive. Others are
   <firstterm>write-back</>, meaning data is sent to the drive at
   some later time. Such caches can be a reliability hazard because the
   memory in the disk controller cache is volatile, and will lose its
   contents in a power failure.  Better controller cards have
   <firstterm>battery-backed unit</> (<acronym>BBU</>) caches, meaning
   the card has a battery that
   maintains power to the cache in case of system power loss.  After power
   is restored the data will be written to the disk drives.
  </para>

  <para>
   And finally, most disk drives have caches. Some are write-through
   while some are write-back, and the same concerns about data loss
   exist for write-back drive caches as exist for disk controller
   caches.  Consumer-grade IDE and SATA drives are particularly likely
   to have write-back caches that will not survive a power failure,
   though <acronym>ATAPI-6</> introduced a drive cache flush command
   (<command>FLUSH CACHE EXT</>) that some file systems use, e.g.
   <acronym>ZFS</>, <acronym>ext4</>.  (The SCSI command
   <command>SYNCHRONIZE CACHE</> has long been available.) Many
   solid-state drives (SSD) also have volatile write-back caches, and
   many do not honor cache flush commands by default.
  </para>

  <para>
   To check write caching on <productname>Linux</> use
   <command>hdparm -I</>;  it is enabled if there is a <literal>*</> next
   to <literal>Write cache</>; <command>hdparm -W</> to turn off
   write caching.  On <productname>FreeBSD</> use
   <application>atacontrol</>.  (For SCSI disks use <ulink
   url="http://sg.danny.cz/sg/sdparm.html"><application>sdparm</></ulink>
   to turn off <literal>WCE</>.)  On <productname>Solaris</> the disk
   write cache is controlled by <ulink
   url="http://www.sun.com/bigadmin/content/submitted/format_utility.jsp"><literal>format
   -e</></ulink>. (The Solaris <acronym>ZFS</> file system is safe with
   disk write-cache enabled because it issues its own disk cache flush
   commands.)  On <productname>Windows</> if <varname>wal_sync_method</>
   is <literal>open_datasync</> (the default), write caching is disabled
   by unchecking <literal>My Computer\Open\{select disk
   drive}\Properties\Hardware\Properties\Policies\Enable write caching on
   the disk</>.  Also on Windows, <literal>fsync</> and
   <literal>fsync_writethrough</> never do write caching.
  </para>

  <para>
   Many file systems that use write barriers (e.g.  <acronym>ZFS</>,
   <acronym>ext4</>) internally use <command>FLUSH CACHE EXT</> or
   <command>SYNCHRONIZE CACHE</> commands to flush data to the platters on
   write-back-enabled drives.  Unfortunately, such write barrier file
   systems behave suboptimally when combined with battery-backed unit
   (<acronym>BBU</>) disk controllers.  In such setups, the synchronize
   command forces all data from the BBU to the disks, eliminating much
   of the benefit of the BBU.  You can run the utility
   <filename>src/tools/fsync</> in the PostgreSQL source tree to see
   if you are affected.  If you are affected, the performance benefits
   of the BBU cache can be regained by turning off write barriers in
   the file system or reconfiguring the disk controller, if that is
   an option.  If write barriers are turned off, make sure the battery
   remains active; a faulty battery can potentially lead to data loss.
   Hopefully file system and disk controller designers will eventually
   address this suboptimal behavior.
  </para>

  <para>
   When the operating system sends a write request to the storage hardware,
   there is little it can do to make sure the data has arrived at a truly
   non-volatile storage area. Rather, it is the
   administrator's responsibility to make certain that all storage components
   ensure data integrity.  Avoid disk controllers that have non-battery-backed
   write caches.  At the drive level, disable write-back caching if the
   drive cannot guarantee the data will be written before shutdown.
   You can test for reliable I/O subsystem behavior using <ulink
   url="http://brad.livejournal.com/2116715.html"><filename>diskchecker.pl</filename></ulink>.
  </para>

  <para>
   Another risk of data loss is posed by the disk platter write
   operations themselves. Disk platters are divided into sectors,
   commonly 512 bytes each.  Every physical read or write operation
   processes a whole sector.
   When a write request arrives at the drive, it might be for 512 bytes,
   1024 bytes, or 8192 bytes, and the process of writing could fail due
   to power loss at any time, meaning some of the 512-byte sectors were
   written, and others were not.  To guard against such failures,
   <productname>PostgreSQL</> periodically writes full page images to
   permanent WAL storage <emphasis>before</> modifying the actual page on
   disk. By doing this, during crash recovery <productname>PostgreSQL</> can
   restore partially-written pages.  If you have a battery-backed disk
   controller or file-system software that prevents partial page writes
   (e.g., ZFS),  you can turn off this page imaging by turning off the
   <xref linkend="guc-full-page-writes"> parameter.
  </para>
 </sect1>

  <sect1 id="wal-intro">
   <title>Write-Ahead Logging (<acronym>WAL</acronym>)</title>

   <indexterm zone="wal">
    <primary>WAL</primary>
   </indexterm>

   <indexterm>
    <primary>transaction log</primary>
    <see>WAL</see>
   </indexterm>

   <para>
    <firstterm>Write-Ahead Logging</firstterm> (<acronym>WAL</acronym>)
    is a standard method for ensuring data integrity.  A detailed
    description can be found in most (if not all) books about
    transaction processing. Briefly, <acronym>WAL</acronym>'s central
    concept is that changes to data files (where tables and indexes
    reside) must be written only after those changes have been logged,
    that is, after log records describing the changes have been flushed
    to permanent storage. If we follow this procedure, we do not need
    to flush data pages to disk on every transaction commit, because we
    know that in the event of a crash we will be able to recover the
    database using the log: any changes that have not been applied to
    the data pages can be redone from the log records.  (This is
    roll-forward recovery, also known as REDO.)
   </para>

   <tip>
    <para>
     Because <acronym>WAL</acronym> restores database file
     contents after a crash, journaled file systems are not necessary for
     reliable storage of the data files or WAL files.  In fact, journaling
     overhead can reduce performance, especially if journaling
     causes file system <emphasis>data</emphasis> to be flushed
     to disk.  Fortunately, data flushing during journaling can
     often be disabled with a file system mount option, e.g.
     <literal>data=writeback</> on a Linux ext3 file system.
     Journaled file systems do improve boot speed after a crash.
    </para>
   </tip>


   <para>
    Using <acronym>WAL</acronym> results in a
    significantly reduced number of disk writes, because only the log
    file needs to be flushed to disk to guarantee that a transaction is
    committed, rather than every data file changed by the transaction.
    The log file is written sequentially,
    and so the cost of syncing the log is much less than the cost of
    flushing the data pages.  This is especially true for servers
    handling many small transactions touching different parts of the data
    store.  Furthermore, when the server is processing many small concurrent
    transactions, one <function>fsync</function> of the log file may
    suffice to commit many transactions.
   </para>

   <para>
    <acronym>WAL</acronym> also makes it possible to support on-line
    backup and point-in-time recovery, as described in <xref
    linkend="continuous-archiving">.  By archiving the WAL data we can support
    reverting to any time instant covered by the available WAL data:
    we simply install a prior physical backup of the database, and
    replay the WAL log just as far as the desired time.  What's more,
    the physical backup doesn't have to be an instantaneous snapshot
    of the database state &mdash; if it is made over some period of time,
    then replaying the WAL log for that period will fix any internal
    inconsistencies.
   </para>
  </sect1>

 <sect1 id="wal-async-commit">
  <title>Asynchronous Commit</title>

   <indexterm>
    <primary>synchronous commit</primary>
   </indexterm>

   <indexterm>
    <primary>asynchronous commit</primary>
   </indexterm>

  <para>
   <firstterm>Asynchronous commit</> is an option that allows transactions
   to complete more quickly, at the cost that the most recent transactions may
   be lost if the database should crash.  In many applications this is an
   acceptable trade-off.
  </para>

  <para>
   As described in the previous section, transaction commit is normally
   <firstterm>synchronous</>: the server waits for the transaction's
   <acronym>WAL</acronym> records to be flushed to permanent storage
   before returning a success indication to the client.  The client is
   therefore guaranteed that a transaction reported to be committed will
   be preserved, even in the event of a server crash immediately after.
   However, for short transactions this delay is a major component of the
   total transaction time.  Selecting asynchronous commit mode means that
   the server returns success as soon as the transaction is logically
   completed, before the <acronym>WAL</acronym> records it generated have
   actually made their way to disk.  This can provide a significant boost
   in throughput for small transactions.
  </para>

  <para>
   Asynchronous commit introduces the risk of data loss. There is a short
   time window between the report of transaction completion to the client
   and the time that the transaction is truly committed (that is, it is
   guaranteed not to be lost if the server crashes).  Thus asynchronous
   commit should not be used if the client will take external actions
   relying on the assumption that the transaction will be remembered.
   As an example, a bank would certainly not use asynchronous commit for
   a transaction recording an ATM's dispensing of cash.  But in many
   scenarios, such as event logging, there is no need for a strong
   guarantee of this kind.
  </para>

  <para>
   The risk that is taken by using asynchronous commit is of data loss,
   not data corruption.  If the database should crash, it will recover
   by replaying <acronym>WAL</acronym> up to the last record that was
   flushed.  The database will therefore be restored to a self-consistent
   state, but any transactions that were not yet flushed to disk will
   not be reflected in that state.  The net effect is therefore loss of
   the last few transactions.  Because the transactions are replayed in
   commit order, no inconsistency can be introduced &mdash; for example,
   if transaction B made changes relying on the effects of a previous
   transaction A, it is not possible for A's effects to be lost while B's
   effects are preserved.
  </para>

  <para>
   The user can select the commit mode of each transaction, so that
   it is possible to have both synchronous and asynchronous commit
   transactions running concurrently.  This allows flexible trade-offs
   between performance and certainty of transaction durability.
   The commit mode is controlled by the user-settable parameter
   <xref linkend="guc-synchronous-commit">, which can be changed in any of
   the ways that a configuration parameter can be set.  The mode used for
   any one transaction depends on the value of
   <varname>synchronous_commit</varname> when transaction commit begins.
  </para>

  <para>
   Certain utility commands, for instance <command>DROP TABLE</>, are
   forced to commit synchronously regardless of the setting of
   <varname>synchronous_commit</varname>.  This is to ensure consistency
   between the server's file system and the logical state of the database.
   The commands supporting two-phase commit, such as <command>PREPARE
   TRANSACTION</>, are also always synchronous.
  </para>

  <para>
   If the database crashes during the risk window between an
   asynchronous commit and the writing of the transaction's
   <acronym>WAL</acronym> records,
   then changes made during that transaction <emphasis>will</> be lost.
   The duration of the
   risk window is limited because a background process (the <quote>WAL
   writer</>) flushes unwritten <acronym>WAL</acronym> records to disk
   every <xref linkend="guc-wal-writer-delay"> milliseconds.
   The actual maximum duration of the risk window is three times
   <varname>wal_writer_delay</varname> because the WAL writer is
   designed to favor writing whole pages at a time during busy periods.
  </para>

  <caution>
   <para>
    An immediate-mode shutdown is equivalent to a server crash, and will
    therefore cause loss of any unflushed asynchronous commits.
   </para>
  </caution>

  <para>
   Asynchronous commit provides behavior different from setting
   <xref linkend="guc-fsync"> = off.
   <varname>fsync</varname> is a server-wide
   setting that will alter the behavior of all transactions.  It disables
   all logic within <productname>PostgreSQL</> that attempts to synchronize
   writes to different portions of the database, and therefore a system
   crash (that is, a hardware or operating system crash, not a failure of
   <productname>PostgreSQL</> itself) could result in arbitrarily bad
   corruption of the database state.  In many scenarios, asynchronous
   commit provides most of the performance improvement that could be
   obtained by turning off <varname>fsync</varname>, but without the risk
   of data corruption.
  </para>

  <para>
   <xref linkend="guc-commit-delay"> also sounds very similar to
   asynchronous commit, but it is actually a synchronous commit method
   (in fact, <varname>commit_delay</varname> is ignored during an
   asynchronous commit).  <varname>commit_delay</varname> causes a delay
   just before a synchronous commit attempts to flush
   <acronym>WAL</acronym> to disk, in the hope that a single flush
   executed by one such transaction can also serve other transactions
   committing at about the same time.  Setting <varname>commit_delay</varname>
   can only help when there are many concurrently committing transactions,
   and it is difficult to tune it to a value that actually helps rather
   than hurt throughput.
  </para>

 </sect1>

 <sect1 id="wal-configuration">
  <title><acronym>WAL</acronym> Configuration</title>

  <para>
   There are several <acronym>WAL</>-related configuration parameters that
   affect database performance. This section explains their use.
   Consult <xref linkend="runtime-config"> for general information about
   setting server configuration parameters.
  </para>

  <para>
   <firstterm>Checkpoints</firstterm><indexterm><primary>checkpoint</></>
   are points in the sequence of transactions at which it is guaranteed
   that the heap and index data files have been updated with all information written before
   the checkpoint.  At checkpoint time, all dirty data pages are flushed to
   disk and a special checkpoint record is written to the log file. 
   (The changes were previously flushed to the <acronym>WAL</acronym> files.)
   In the event of a crash, the crash recovery procedure looks at the latest
   checkpoint record to determine the point in the log (known as the redo
   record) from which it should start the REDO operation.  Any changes made to
   data files before that point are guaranteed to be already on disk.  Hence, after
   a checkpoint, log segments preceding the one containing
   the redo record are no longer needed and can be recycled or removed. (When
   <acronym>WAL</acronym> archiving is being done, the log segments must be
   archived before being recycled or removed.)
  </para>

  <para>
   The checkpoint requirement of flushing all dirty data pages to disk
   can cause a significant I/O load.  For this reason, checkpoint
   activity is throttled so I/O begins at checkpoint start and completes
   before the next checkpoint starts;  this minimizes performance
   degradation during checkpoints.
  </para>

  <para>
   The server's background writer process automatically performs
   a checkpoint every so often.  A checkpoint is created every <xref
   linkend="guc-checkpoint-segments"> log segments, or every <xref
   linkend="guc-checkpoint-timeout"> seconds, whichever comes first.
   The default settings are 3 segments and 300 seconds (5 minutes), respectively.
   It is also possible to force a checkpoint by using the SQL command
   <command>CHECKPOINT</command>.
  </para>

  <para>
   Reducing <varname>checkpoint_segments</varname> and/or
   <varname>checkpoint_timeout</varname> causes checkpoints to occur
   more often. This allows faster after-crash recovery (since less work
   will need to be redone). However, one must balance this against the
   increased cost of flushing dirty data pages more often. If
   <xref linkend="guc-full-page-writes"> is set (as is the default), there is
   another factor to consider. To ensure data page consistency,
   the first modification of a data page after each checkpoint results in
   logging the entire page content. In that case,
   a smaller checkpoint interval increases the volume of output to the WAL log,
   partially negating the goal of using a smaller interval,
   and in any case causing more disk I/O.
  </para>

  <para>
   Checkpoints are fairly expensive, first because they require writing
   out all currently dirty buffers, and second because they result in
   extra subsequent WAL traffic as discussed above.  It is therefore
   wise to set the checkpointing parameters high enough that checkpoints
   don't happen too often.  As a simple sanity check on your checkpointing
   parameters, you can set the <xref linkend="guc-checkpoint-warning">
   parameter.  If checkpoints happen closer together than
   <varname>checkpoint_warning</> seconds,
   a message will be output to the server log recommending increasing
   <varname>checkpoint_segments</varname>.  Occasional appearance of such
   a message is not cause for alarm, but if it appears often then the
   checkpoint control parameters should be increased. Bulk operations such
   as large <command>COPY</> transfers might cause a number of such warnings
   to appear if you have not set <varname>checkpoint_segments</> high
   enough.
  </para>

  <para>
   To avoid flooding the I/O system with a burst of page writes,
   writing dirty buffers during a checkpoint is spread over a period of time.
   That period is controlled by
   <xref linkend="guc-checkpoint-completion-target">, which is
   given as a fraction of the checkpoint interval.
   The I/O rate is adjusted so that the checkpoint finishes when the
   given fraction of <varname>checkpoint_segments</varname> WAL segments
   have been consumed since checkpoint start, or the given fraction of
   <varname>checkpoint_timeout</varname> seconds have elapsed,
   whichever is sooner.  With the default value of 0.5,
   <productname>PostgreSQL</> can be expected to complete each checkpoint
   in about half the time before the next checkpoint starts.  On a system
   that's very close to maximum I/O throughput during normal operation,
   you might want to increase <varname>checkpoint_completion_target</varname>
   to reduce the I/O load from checkpoints.  The disadvantage of this is that
   prolonging checkpoints affects recovery time, because more WAL segments
   will need to be kept around for possible use in recovery.  Although
   <varname>checkpoint_completion_target</varname> can be set as high as 1.0,
   it is best to keep it less than that (perhaps 0.9 at most) since
   checkpoints include some other activities besides writing dirty buffers.
   A setting of 1.0 is quite likely to result in checkpoints not being
   completed on time, which would result in performance loss due to
   unexpected variation in the number of WAL segments needed.
  </para>

  <para>
   There will always be at least one WAL segment file, and will normally
   not be more than (2 + <varname>checkpoint_completion_target</varname>) * <varname>checkpoint_segments</varname> + 1
   or <varname>checkpoint_segments</> + <xref linkend="guc-wal-keep-segments"> + 1
   files.  Each segment file is normally 16 MB (though this size can be
   altered when building the server).  You can use this to estimate space
   requirements for <acronym>WAL</acronym>.
   Ordinarily, when old log segment files are no longer needed, they
   are recycled (renamed to become the next segments in the numbered
   sequence). If, due to a short-term peak of log output rate, there
   are more than 3 * <varname>checkpoint_segments</varname> + 1
   segment files, the unneeded segment files will be deleted instead
   of recycled until the system gets back under this limit.
  </para>

  <para>
   In archive recovery or standby mode, the server periodically performs
   <firstterm>restartpoints</><indexterm><primary>restartpoint</></>
   which are similar to checkpoints in normal operation: the server forces
   all its state to disk, updates the <filename>pg_control</> file to
   indicate that the already-processed WAL data need not be scanned again,
   and then recycles any old log segment files in <filename>pg_xlog</>
   directory. A restartpoint is triggered if at least one checkpoint record
   has been replayed and <varname>checkpoint_timeout</> seconds have passed
   since last restartpoint. In standby mode, a restartpoint is also triggered
   if <varname>checkpoint_segments</> log segments have been replayed since
   last restartpoint and at least one checkpoint record has been replayed.
   Restartpoints can't be performed more frequently than checkpoints in the
   master because restartpoints can only be performed at checkpoint records.
  </para>

  <para>
   There are two commonly used internal <acronym>WAL</acronym> functions:
   <function>LogInsert</function> and <function>LogFlush</function>.
   <function>LogInsert</function> is used to place a new record into
   the <acronym>WAL</acronym> buffers in shared memory. If there is no
   space for the new record, <function>LogInsert</function> will have
   to write (move to kernel cache) a few filled <acronym>WAL</acronym>
   buffers. This is undesirable because <function>LogInsert</function>
   is used on every database low level modification (for example, row
   insertion) at a time when an exclusive lock is held on affected
   data pages, so the operation needs to be as fast as possible.  What
   is worse, writing <acronym>WAL</acronym> buffers might also force the
   creation of a new log segment, which takes even more
   time. Normally, <acronym>WAL</acronym> buffers should be written
   and flushed by a <function>LogFlush</function> request, which is
   made, for the most part, at transaction commit time to ensure that
   transaction records are flushed to permanent storage. On systems
   with high log output, <function>LogFlush</function> requests might
   not occur often enough to prevent <function>LogInsert</function>
   from having to do writes.  On such systems
   one should increase the number of <acronym>WAL</acronym> buffers by
   modifying the configuration parameter <xref
   linkend="guc-wal-buffers">.  The default number of <acronym>WAL</acronym>
   buffers is 8.  Increasing this value will
   correspondingly increase shared memory usage.  When
   <xref linkend="guc-full-page-writes"> is set and the system is very busy,
   setting this value higher will help smooth response times during the
   period immediately following each checkpoint.
  </para>

  <para>
   The <xref linkend="guc-commit-delay"> parameter defines for how many
   microseconds the server process will sleep after writing a commit
   record to the log with <function>LogInsert</function> but before
   performing a <function>LogFlush</function>. This delay allows other
   server processes to add their commit records to the log so as to have all
   of them flushed with a single log sync. No sleep will occur if
   <xref linkend="guc-fsync">
   is not enabled, or if fewer than <xref linkend="guc-commit-siblings">
   other sessions are currently in active transactions; this avoids
   sleeping when it's unlikely that any other session will commit soon.
   Note that on most platforms, the resolution of a sleep request is
   ten milliseconds, so that any nonzero <varname>commit_delay</varname>
   setting between 1 and 10000 microseconds would have the same effect.
   Good values for these parameters are not yet clear; experimentation
   is encouraged.
  </para>

  <para>
   The <xref linkend="guc-wal-sync-method"> parameter determines how
   <productname>PostgreSQL</productname> will ask the kernel to force
    <acronym>WAL</acronym> updates out to disk.
   All the options should be the same in terms of reliability,
   but it's quite platform-specific which one will be the fastest.
   Note that this parameter is irrelevant if <varname>fsync</varname>
   has been turned off.
  </para>

  <para>
   Enabling the <xref linkend="guc-wal-debug"> configuration parameter
   (provided that <productname>PostgreSQL</productname> has been
   compiled with support for it) will result in each
   <function>LogInsert</function> and <function>LogFlush</function>
   <acronym>WAL</acronym> call being logged to the server log. This
   option might be replaced by a more general mechanism in the future.
  </para>
 </sect1>

 <sect1 id="wal-internals">
  <title>WAL Internals</title>

  <para>
   <acronym>WAL</acronym> is automatically enabled; no action is
   required from the administrator except ensuring that the
   disk-space requirements for the <acronym>WAL</acronym> logs are met,
   and that any necessary tuning is done (see <xref
   linkend="wal-configuration">).
  </para>

  <para>
   <acronym>WAL</acronym> logs are stored in the directory
   <filename>pg_xlog</filename> under the data directory, as a set of
   segment files, normally each 16 MB in size (but the size can be changed
   by altering the <option>--with-wal-segsize</> configure option when
   building the server).  Each segment is divided into pages, normally
   8 kB each (this size can be changed via the <option>--with-wal-blocksize</>
   configure option).  The log record headers are described in
   <filename>access/xlog.h</filename>; the record content is dependent
   on the type of event that is being logged.  Segment files are given
   ever-increasing numbers as names, starting at
   <filename>000000010000000000000000</filename>.  The numbers do not wrap,
   but it will take a very, very long time to exhaust the
   available stock of numbers.
  </para>

  <para>
   It is advantageous if the log is located on a different disk from the
   main database files.  This can be achieved by moving the
   <filename>pg_xlog</filename> directory to another location (while the server
   is shut down, of course) and creating a symbolic link from the
   original location in the main data directory to the new location.
  </para>

  <para>
   The aim of <acronym>WAL</acronym> is to ensure that the log is
   written before database records are altered, but this can be subverted by
   disk drives<indexterm><primary>disk drive</></> that falsely report a
   successful write to the kernel,
   when in fact they have only cached the data and not yet stored it
   on the disk.  A power failure in such a situation might lead to
   irrecoverable data corruption.  Administrators should try to ensure
   that disks holding <productname>PostgreSQL</productname>'s
   <acronym>WAL</acronym> log files do not make such false reports.
  </para>

  <para>
   After a checkpoint has been made and the log flushed, the
   checkpoint's position is saved in the file
   <filename>pg_control</filename>. Therefore, at the start of recovery,
   the server first reads <filename>pg_control</filename> and
   then the checkpoint record; then it performs the REDO operation by
   scanning forward from the log position indicated in the checkpoint
   record.  Because the entire content of data pages is saved in the
   log on the first page modification after a checkpoint (assuming
   <xref linkend="guc-full-page-writes"> is not disabled), all pages
   changed since the checkpoint will be restored to a consistent
   state.
  </para>

  <para>
   To deal with the case where <filename>pg_control</filename> is
   corrupt, we should support the possibility of scanning existing log
   segments in reverse order &mdash; newest to oldest &mdash; in order to find the
   latest checkpoint.  This has not been implemented yet.
   <filename>pg_control</filename> is small enough (less than one disk page)
   that it is not subject to partial-write problems, and as of this writing
   there have been no reports of database failures due solely to the inability
   to read <filename>pg_control</filename> itself.  So while it is
   theoretically a weak spot, <filename>pg_control</filename> does not
   seem to be a problem in practice.
  </para>
 </sect1>
</chapter>
@


1.71
log
@Spelling fixes
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.70 2010/07/16 11:35:40 heikki Exp $ -->
d119 1
a119 1
   url="http://brad.livejournal.com/2116715.html">diskchecker.pl</ulink>.
@


1.70
log
@Fix typo spotted by Thom Brown.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.69 2010/07/16 11:20:23 heikki Exp $ -->
d94 1
a94 1
   <command>SYNCHRONIZE CACHE</> commands to flush data to the platers on
@


1.69
log
@Add a paragraph explaining what restartpoints are. Mention that
wal_keep_segments does not take effect during recovery.

Fujii Masao
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.68 2010/07/08 16:44:12 momjian Exp $ -->
d474 1
a474 1
   if <varname>checkoint_segments</> log segments have been replayed since
@


1.68
log
@Doc change:  effected -> affected, per correction from Matthew Wakeling
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.67 2010/07/07 14:42:09 momjian Exp $ -->
d452 1
d465 16
@


1.68.2.1
log
@Add a paragraph explaining what restartpoints are. Mention that
wal_keep_segments does not take effect during recovery.

Fujii Masao
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.68 2010/07/08 16:44:12 momjian Exp $ -->
a451 1
   or <varname>checkpoint_segments</> + <xref linkend="guc-wal-keep-segments"> + 1
a463 16
   In archive recovery or standby mode, the server periodically performs
   <firstterm>restartpoints</><indexterm><primary>restartpoint</></>
   which are similar to checkpoints in normal operation: the server forces
   all its state to disk, updates the <filename>pg_control</> file to
   indicate that the already-processed WAL data need not be scanned again,
   and then recycles any old log segment files in <filename>pg_xlog</>
   directory. A restartpoint is triggered if at least one checkpoint record
   has been replayed and <varname>checkpoint_timeout</> seconds have passed
   since last restartpoint. In standby mode, a restartpoint is also triggered
   if <varname>checkoint_segments</> log segments have been replayed since
   last restartpoint and at least one checkpoint record has been replayed.
   Restartpoints can't be performed more frequently than checkpoints in the
   master because restartpoints can only be performed at checkpoint records.
  </para>

  <para>
@


1.68.2.2
log
@Fix typo spotted by Thom Brown.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.68.2.1 2010/07/16 11:21:46 heikki Exp $ -->
d474 1
a474 1
   if <varname>checkpoint_segments</> log segments have been replayed since
@


1.68.2.3
log
@Spelling fixes
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.68.2.2 2010/07/16 11:36:10 heikki Exp $ -->
d94 1
a94 1
   <command>SYNCHRONIZE CACHE</> commands to flush data to the platters on
@


1.68.2.4
log
@Spell and markup checking
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.68.2.3 2010/07/27 19:01:04 petere Exp $ -->
d119 1
a119 1
   url="http://brad.livejournal.com/2116715.html"><filename>diskchecker.pl</filename></ulink>.
@


1.67
log
@Document the interaction of write-barrier-enabled file systems, and BBU
caches, per June email thread.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.66 2010/04/13 14:15:25 momjian Exp $ -->
d101 1
a101 1
   if you are effected.  If you are effected, the performance benefits
@


1.66
log
@Add "SSD" acronym mention for solid state drive mention.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.65 2010/03/17 17:12:31 petere Exp $ -->
d51 2
a52 1
   <firstterm>battery-backed</> caches, meaning the card has a battery that
d59 13
a71 8
   while some are write-back, and the
   same concerns about data loss exist for write-back drive caches as
   exist for disk controller caches.  Consumer-grade IDE and SATA drives are
   particularly likely to have write-back caches that will not survive a
   power failure, though <acronym>ATAPI-6</> introduced a drive cache
   flush command (FLUSH CACHE EXT) that some file systems use, e.g. <acronym>ZFS</>.
   Many solid-state drives (SSD) also have volatile write-back
   caches, and many do not honor cache flush commands by default.
d92 19
@


1.65
log
@Update broken and permanently moved links
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.64 2010/02/27 20:16:17 momjian Exp $ -->
d64 1
a64 1
   Many solid-state drives also have volatile write-back
@


1.64
log
@Document ATAPI FLUSH CACHE EXT.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.63 2010/02/27 01:39:46 momjian Exp $ -->
d71 1
a71 1
   url="http://sg.torque.net/sg/sdparm.html"><application>sdparm</></ulink>
@


1.63
log
@Document ATAPI drive flush command, and mention SSD drives.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.62 2010/02/20 18:28:37 momjian Exp $ -->
d63 1
a63 1
   flush command that some file systems use, e.g. <acronym>ZFS</>.
@


1.62
log
@Document that many solid-state drives have volatile write-back caches.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.61 2010/02/03 17:25:06 momjian Exp $ -->
d62 5
a66 2
   power failure.  Many solid-state drives also have volatile write-back
   caches.  To check write caching on <productname>Linux</> use
@


1.61
log
@Proofreading improvements for the Administration documentation book.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.60 2009/11/28 16:21:31 momjian Exp $ -->
d62 2
a63 1
   power failure.  To check write caching on <productname>Linux</> use
@


1.60
log
@fsync test tools

Add link to exteran fsync testing script and our fsync test tool.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.59 2009/04/09 16:20:50 momjian Exp $ -->
d45 3
a47 3
   these caches are <firstterm>write-through</>, meaning writes are passed
   along to the drive as soon as they arrive. Others are
   <firstterm>write-back</>, meaning data is passed on to the drive at
d64 1
a64 1
   to <literal>Write cache</>.  <command>hdparm -W</> to turn off
d82 1
a82 1
   When the operating system sends a write request to the disk hardware,
d85 1
a85 1
   administrator's responsibility to be sure that all storage components
d103 1
a103 1
   permanent storage <emphasis>before</> modifying the actual page on
d107 1
a107 1
   (e.g., ReiserFS 4),  you can turn off this page imaging by using the
d143 1
a143 1
     contents after a crash, journaled filesystems are not necessary for
d148 1
a148 1
     often be disabled with a filesystem mount option, e.g.
d311 1
a311 1
   than hurting throughput.
d329 1
a329 1
   that the data files have been updated with all information written before
d352 1
a352 1
   The server's background writer process will automatically perform
d356 1
a356 1
   The default settings are 3 segments and 300 seconds respectively.
d363 1
a363 1
   <varname>checkpoint_timeout</varname> causes checkpoints to be done
d472 1
a472 1
   is not enabled, nor if fewer than <xref linkend="guc-commit-siblings">
d486 1
a486 1
   All the options should be the same as far as reliability goes,
d524 2
a525 2
   <filename>000000010000000000000000</filename>.  The numbers do not wrap, at
   present, but it should take a very very long time to exhaust the
d530 3
a532 3
   It is of advantage if the log is located on another disk than the
   main database files.  This can be achieved by moving the directory
   <filename>pg_xlog</filename> to another location (while the server
d538 2
a539 2
   The aim of <acronym>WAL</acronym>, to ensure that the log is
   written before database records are altered, can be subverted by
d543 1
a543 1
   on the disk.  A power failure in such a situation might still lead to
d552 2
a553 2
   <filename>pg_control</filename>. Therefore, when recovery is to be
   done, the server first reads <filename>pg_control</filename> and
d565 1
a565 1
   corrupted, we should support the possibility of scanning existing log
d570 1
a570 1
   there have been no reports of database failures due solely to inability
@


1.59
log
@Improve documentation about how checkpoint spreads I/O activity.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.58 2009/01/15 00:34:25 momjian Exp $ -->
d89 2
@


1.58
log
@Add documentation mention that full page writes have to be enabled for
full pages to be restored during recovery.

Per private report from Michael Renner
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.57 2008/12/18 22:34:21 momjian Exp $ -->
d329 2
a330 1
   disk and a special checkpoint record is written to the log file.
d334 2
a335 2
   data files before that point are known to be already on disk.  Hence, after
   a checkpoint has been made, any log segments preceding the one containing
d342 8
@


1.57
log
@Add "not" to wal journaling text.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.56 2008/12/18 22:21:16 momjian Exp $ -->
d546 2
a547 1
   log on the first page modification after a checkpoint, all pages
@


1.56
log
@Clarify documentation that journaling is not required for WAL or data files.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.55 2008/12/10 11:05:49 momjian Exp $ -->
d141 1
a141 1
     contents after a crash, journaled filesystems are necessary for
@


1.55
log
@Update journaling performance docs based on comments by Michael Renner.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.54 2008/12/06 21:34:27 momjian Exp $ -->
d141 2
a142 2
     contents after a crash, it is not necessary to use a
     journaled filesystem for reliability.  In fact, journaling
@


1.54
log
@Document that non-data journaling is a recommended mount option.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.53 2008/05/02 19:52:37 tgl Exp $ -->
d142 7
a148 7
     journaled filesystem;  in fact, journaling overhead can
     reduce performance.  For best performance, turn off
     <emphasis>data</emphasis> journaling as a filesystem mount
     option, e.g. use <literal>data=writeback</> on Linux.
     Meta-data journaling (e.g.  file creation and directory
     modification) is still desirable for faster rebooting after
     a crash.
@


1.53
log
@Extend yesterday's patch making BLCKSZ and RELSEG_SIZE configurable to also
let XLOG_BLCKSZ and XLOG_SEG_SIZE be set via configure.  Per a proposal by
Mark Wong, though I thought it better to call the switches after "wal" rather
than "xlog".
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.52 2007/12/29 17:55:07 momjian Exp $ -->
d138 15
@


1.52
log
@Document how to control the disk write cache on Solaris.

Zdenek Kotala
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.51 2007/12/16 14:05:12 momjian Exp $ -->
d490 5
a494 2
   segment files, normally each 16 MB in size.  Each segment is divided into
   pages, normally 8 kB each.  The log record headers are described in
@


1.51
log
@Windows write-cache wording improvement.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.50 2007/12/16 14:03:32 momjian Exp $ -->
d68 8
a75 4
   to turn off <literal>WCE</>.)  On <productname>Windows</> if
   <varname>wal_sync_method</> is <literal>open_datasync</> (the default),
   write caching is disabled by unchecking <literal>My
   Computer\Open\{select disk
@


1.50
log
@Update write-cache docs to mention windows behavior for various
fsync_methods.

Magnus.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.49 2007/12/10 22:08:36 momjian Exp $ -->
d74 1
a74 1
   <literal>fsync_writethrough</> never use write caching.
@


1.49
log
@Wording improvement for write-cache disabling instructions.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.48 2007/12/10 14:51:10 momjian Exp $ -->
d68 4
a71 2
   to turn off <literal>WCE</>.)  On <productname>Windows</> write caching is
   disabled by unchecking <literal>My Computer\Open\{select disk
d73 2
a74 1
   the disk</>.
@


1.48
log
@Write cache doc cleanups

Greg Smith
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.47 2007/12/10 14:05:05 momjian Exp $ -->
d69 1
a69 1
   controlled by <literal>My Computer\Open\{select disk
@


1.47
log
@Document how to turn off disk write cache on popular operating systems.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.46 2007/11/28 15:42:31 petere Exp $ -->
d60 1
a60 1
   exist for disk controller caches.  Consumer-grade IDE drives are
d64 1
a64 1
   to <literal>Write cache</>.  <command>hdparm -W</> can to turn off
@


1.46
log
@spell checker run
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.45 2007/08/01 22:45:07 tgl Exp $ -->
d61 11
a71 2
   particularly likely to contain write-back caches that will not
   survive a power failure.
@


1.45
log
@Support an optional asynchronous commit mode, in which we don't flush WAL
before reporting a transaction committed.  Data consistency is still
guaranteed (unlike setting fsync = off), but a crash may lose the effects
of the last few transactions.  Patch by Simon, some editorialization by Tom.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.44 2007/06/28 00:02:37 tgl Exp $ -->
d165 1
a165 1
   acceptable tradeoff.
d213 1
a213 1
   transactions running concurrently.  This allows flexible tradeoffs
d226 1
a226 1
   between the server's filesystem and the logical state of the database.
d237 1
a237 1
   risk window is limited because a background process (the <quote>wal
d241 1
a241 1
   <varname>wal_writer_delay</varname> because the wal writer is
@


1.44
log
@Implement "distributed" checkpoints in which the checkpoint I/O is spread
over a fairly long period of time, rather than being spat out in a burst.
This happens only for background checkpoints carried out by the bgwriter;
other cases, such as a shutdown checkpoint, are still done at full speed.

Remove the "all buffers" scan in the bgwriter, and associated stats
infrastructure, since this seems no longer very useful when the checkpoint
itself is properly throttled.

Original patch by Itagaki Takahiro, reworked by Heikki Linnakangas,
and some minor API editorialization by me.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.43 2007/01/31 20:56:19 momjian Exp $ -->
d7 1
a7 1
  This chapter explain how the Write-Ahead Log is used to obtain
d74 1
a74 1
  
d89 1
a89 1
   (e.g., ReiserFS 4),  you can turn off this page imaging by using the 
d93 1
a93 1
 
d108 1
a108 1
    is a standard approach to transaction logging.  Its detailed
d113 1
a113 1
    that is, when log records describing the changes have been flushed
d123 1
a123 1
    A major benefit of using <acronym>WAL</acronym> is a
d125 3
a127 5
    file needs to be flushed to disk at the time of transaction
    commit, rather than every data file changed by the transaction.
    In multiuser environments, commits of many transactions
    can be accomplished with a single <function>fsync</function> of
    the log file. Furthermore, the log file is written sequentially,
d129 1
a129 1
    flushing the data pages.   This is especially true for servers
d131 3
a133 1
    store.
d150 133
d324 4
a327 4
   increased cost of flushing dirty data pages more often. If 
   <xref linkend="guc-full-page-writes"> is set (as is the default), there is 
   another factor to consider. To ensure data page consistency, 
   the first modification of a data page after each checkpoint results in 
d330 1
a330 1
   partially negating the goal of using a smaller interval, 
d342 2
a343 2
   <varname>checkpoint_warning</> seconds, 
   a message will be output to the server log recommending increasing 
d416 3
a418 3
   correspondingly increase shared memory usage.  When 
   <xref linkend="guc-full-page-writes"> is set and the system is very busy, 
   setting this value higher will help smooth response times during the 
d443 1
a443 1
    <acronym>WAL</acronym> updates out to disk. 
d496 1
a496 1
   successful write to the kernel, 
@


1.43
log
@Update documentation on may/can/might:

Standard English uses "may", "can", and "might" in different ways:

        may - permission, "You may borrow my rake."

        can - ability, "I can lift that log."

        might - possibility, "It might rain today."

Unfortunately, in conversational English, their use is often mixed, as
in, "You may use this variable to do X", when in fact, "can" is a better
choice.  Similarly, "It may crash" is better stated, "It might crash".

Also update two error messages mentioned in the documenation to match.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.42 2006/11/25 22:44:48 petere Exp $ -->
d220 28
a247 2
   There will be at least one WAL segment file, and will normally
   not be more than 2 * <varname>checkpoint_segments</varname> + 1
d254 1
a254 1
   are more than 2 * <varname>checkpoint_segments</varname> + 1
@


1.42
log
@Correct misspellings of kB.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.41 2006/10/23 18:10:32 petere Exp $ -->
d43 1
a43 1
   Next, there may be a cache in the disk drive controller; this is
d109 1
a109 1
    description may be found in most (if not all) books about
d128 1
a128 1
    may be accomplished with a single <function>fsync</function> of
d214 1
a214 1
   as large <command>COPY</> transfers may cause a number of such warnings
d244 1
a244 1
   is worse, writing <acronym>WAL</acronym> buffers may also force the
d250 1
a250 1
   with high log output, <function>LogFlush</function> requests may
d297 1
a297 1
   option may be replaced by a more general mechanism in the future.
d327 1
a327 1
   main database files.  This may be achieved by moving the directory
d335 1
a335 1
   written before database records are altered, may be subverted by
d339 1
a339 1
   on the disk.  A power failure in such a situation may still lead to
@


1.41
log
@Spellchecking and such
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.40 2006/09/16 00:30:16 momjian Exp $ -->
d316 1
a316 1
   pages, normally 8 KB each.  The log record headers are described in
@


1.40
log
@Remove emacs info from footer of SGML files.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.39 2006/03/03 22:02:08 momjian Exp $ -->
d88 2
a89 2
   controller or file-system software (e.g., Reiser4) that prevents partial
   page writes,  you can turn off this page imaging by using the 
@


1.39
log
@Rename Online Backup to Continuous Archiving.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.38 2005/11/04 23:14:02 petere Exp $ -->
a371 17

<!-- Keep this comment at the end of the file
Local variables:
mode:sgml
sgml-omittag:nil
sgml-shorttag:t
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:t
sgml-parent-document:nil
sgml-default-dtd-file:"./reference.ced"
sgml-exposed-tags:nil
sgml-local-catalogs:("/usr/lib/sgml/catalog")
sgml-local-ecat-files:nil
End:
-->
@


1.38
log
@Spellchecking run, final cleanups
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.37 2005/10/22 21:56:07 tgl Exp $ -->
d139 1
a139 1
    linkend="backup-online">.  By archiving the WAL data we can support
@


1.37
log
@Copy-editing for recent documentation changes relevant to WAL,
full_page_writes, etc.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.36 2005/10/13 17:32:42 momjian Exp $ -->
d3 2
a4 2
<chapter id="reliability">
 <title>Reliability</title>
d6 21
a26 12
  <para>
   Reliability is a major feature of any serious database system, and
   <productname>PostgreSQL</> does everything possible to guarantee
   reliable operation. One aspect of reliable operation is that all data
   recorded by a committed transaction should be stored in a non-volatile area
   that is safe from power loss, operating system failure, and hardware
   failure (except failure of the non-volatile area itself, of course).
   Successfully writing the data to the computer's permanent storage
   (disk drive or equivalent) ordinarily meets this requirement.
   In fact, even if a computer is fatally damaged, if
   the disk drives survive they can be moved to another computer with
   similar hardware and all committed transactions will remain intact.
d88 1
a88 1
   controller or filesystem software (e.g., Reiser4) that prevents partial
d92 1
d94 1
a94 6
  <para>
   The following sections explain how the Write-Ahead Log is used to 
   obtain efficient, reliable operation.
  </para>

  <sect1 id="wal">
@


1.36
log
@Some additional doc changes based around compression of page images in
WAL and the interaction of the new full_page_writes parameter with PITR.

The too-small WAL first sect1 has been merged with the one following
sect1 for clarity.

Some minor comments have been made in the WAL config section also.

Passes SGML make and proofread for typos.
Files changed:
patching file doc/src/sgml/backup.sgml
patching file doc/src/sgml/config.sgml
patching file doc/src/sgml/wal.sgml

Simon Riggs
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.35 2005/10/01 01:42:43 momjian Exp $ -->
d10 1
a10 1
   recorded by a transaction should be stored in a non-volatile area
d12 4
a15 4
   failure (unrelated to the non-volatile area itself). To accomplish
   this, <productname>PostgreSQL</> uses the magnetic platters of modern
   disk drives for permanent storage that is immune to the failures
   listed above. In fact, even if a computer is fatally damaged, if
d24 3
a26 3
   between the computer's main memory and the disk drive platters.
   First, there is the operating system kernel cache, which caches
   frequently requested disk blocks and delays disk writes. Fortunately,
d28 3
a30 3
   the kernel cache to disk, and <productname>PostgreSQL</> uses those
   features. In fact, the <xref linkend="guc-wal-sync-method"> parameter
   controls how this is done.
d32 1
d34 3
a36 3
   Secondly, there is an optional disk drive controller cache,
   particularly popular on <acronym>RAID</> controller cards. Some of
   these caches are <literal>write-through</>, meaning writes are passed
d38 7
a44 7
   <literal>write-back</>, meaning data is passed on to the drive at
   some later time. Such caches can be a reliability problem because the
   disk controller card cache is volatile, unlike the disk driver
   platters, unless the disk drive controller has a battery-backed
   cache, meaning the card has a battery that maintains power to the
   cache in case of server power loss. When the disk drives are later
   accessible, the data is written to the drives.
d49 1
a49 1
   (typically SCSI), and some are write-back(typically IDE), and the
d51 9
a59 5
   exist for disk controller caches. To have reliability, all
   storage subsystems must be reliable in their storage characteristics.
   When the operating system sends a write request to the drive platters,
   there is little it can do to make sure the data has arrived at a
   non-volatile store area on the system. Rather, it is the
d61 3
a63 1
   have reliable characteristics.
d67 4
a70 2
   One other area of potential data loss are the disk platter writes
   themselves. Disk platters are internally made up of 512-byte sectors.
d74 1
a74 4
   written, and others were not, or the first half of a 512-byte sector
   has new data, and the remainder has the original data. Obviously, on
   startup, <productname>PostgreSQL</> would not be able to deal with
   these partially written cases. To guard against that,
d78 4
a81 6
   restore partially-written pages. If you have a battery-backed disk
   controller or filesystem (e.g. Reiser4) that prevents partial page writes, 
   you can turn off this page imaging by using the 
   <xref linkend="guc-full-page-writes"> parameter. This parameter has no 
   effect on the successful use of Point in Time Recovery (PITR), 
   described in <xref linkend="backup-online">.
d118 1
a118 5
    WAL brings three major benefits:
   </para>

   <para>
    The first major benefit of using <acronym>WAL</acronym> is a
d132 1
a132 24
    The next benefit is crash recovery protection. The truth is
    that, before <acronym>WAL</acronym> was introduced back in release 7.1,
    <productname>PostgreSQL</productname> was never able to guarantee
    consistency in the case of a crash.  Now, 
    <acronym>WAL</acronym> protects fully against the following problems:

    <orderedlist>
     <listitem>
      <simpara>index rows pointing to nonexistent table rows</simpara>
     </listitem>

     <listitem>
      <simpara>index rows lost in split operations</simpara>
     </listitem>

     <listitem>
      <simpara>totally corrupted table or index page content, because
      of partially written data pages</simpara>
     </listitem>
    </orderedlist>
   </para>

   <para>
    Finally, <acronym>WAL</acronym> makes it possible to support on-line
d149 1
a149 1
   There are several <acronym>WAL</acronym>-related configuration parameters that
d158 1
a158 1
   that the data files have been updated with all information logged before
d160 9
a168 8
   disk and a special checkpoint record is written to the log file. As a
   result, in the event of a crash, the crash recovery procedure knows from
   what point in the log (known as the redo record) it should start the
   REDO operation, since any changes made to data files before that point
   are already on disk. After a checkpoint has been made, any log segments
   written before the redo record are no longer needed and can be recycled
   or removed. (When <acronym>WAL</acronym> archiving is being done, the
   log segments must be archived before being recycled or removed.)
d187 1
a187 1
   <xref linkend="guc-full-page-writes"> is set (the default), there is 
d209 3
a211 2
   as a COPY, INSERT SELECT etc. may cause a number of such warnings if you
   do not set <xref linkend="guc-checkpoint-segments"> high enough.
d255 1
a255 2
   period immediately following each checkpoint.  As a guide, a setting of 1024 
   would be considered to be high.
d291 1
a291 2
   <acronym>WAL</acronym> call being logged to the server log. The output
   is too verbose for use as a guide to performance tuning. This
a320 9
   The <acronym>WAL</acronym> buffers and control structure are in
   shared memory and are handled by the server child processes; they
   are protected by lightweight locks.  The demand on shared memory is
   dependent on the number of buffers.  The default size of the
   <acronym>WAL</acronym> buffers is 8 buffers of 8 kB each, or 64 kB
   total.
  </para>

  <para>
@


1.35
log
@Adjust wording.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.34 2005/09/29 02:29:48 momjian Exp $ -->
d15 1
a15 1
   listed above. In fact, a computer can be completely destroyed, but if
d71 1
a71 1
   disk. By doing this, during recovery <productname>PostgreSQL</> can
d73 5
a77 3
   controller that prevents partial page writes, you can turn off this
   page imaging by using the <xref linkend="guc-full-page-writes">
   parameter.
a111 1
  </sect1>
d113 3
a115 6
  <sect1 id="wal-benefits">
   <title>Benefits of Write-Ahead Logging</title>

   <indexterm zone="wal-benefits">
    <primary>fsync</primary>
   </indexterm>
d132 2
a133 2
    The next benefit is consistency of the data pages. The truth is
    that, before <acronym>WAL</acronym>,
d135 2
a136 2
    consistency in the case of a crash.  Before
    <acronym>WAL</acronym>, any crash during writing could result in:
a151 7

    Problems with indexes (problems 1 and 2) could possibly have been
    fixed by additional <function>fsync</function> calls, but it is
    not obvious how to handle the last case without
    <acronym>WAL</acronym>.  <acronym>WAL</acronym> saves the entire data
    page content in the log if that is required to ensure page
    consistency for after-crash recovery.
d208 8
a215 6
   increased cost of flushing dirty data pages more often. In addition,
   to ensure data page consistency, the first modification of a data
   page after each checkpoint results in logging the entire page
   content. Thus a smaller checkpoint interval increases the volume of
   output to the WAL log, partially negating the goal of using a smaller
   interval, and in any case causing more disk I/O.
d230 3
a232 1
   checkpoint control parameters should be increased.
d250 1
a250 1
   There are two commonly used <acronym>WAL</acronym> functions:
d273 5
a277 3
   correspondingly increase shared memory usage.  (It should be noted
   that there is presently little evidence to suggest that increasing
   <varname>wal_buffers</> beyond the default is worthwhile.)
d313 2
a314 1
   <acronym>WAL</acronym> call being logged to the server log. This
@


1.34
log
@Remove acronym in one title.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.33 2005/09/28 18:40:28 momjian Exp $ -->
d79 2
a80 2
   The following sections into detail about how the Write-Ahead Log
   is used to obtain efficient, reliable operation.
@


1.33
log
@Fix typo.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.32 2005/09/28 18:18:02 momjian Exp $ -->
d113 1
a113 1
   <title>Benefits of <acronym>WAL</acronym></title>
@


1.32
log
@Add section on reliable operation, talking about caching and storage
subsystem reliability.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.31 2004/11/15 06:32:14 neilc Exp $ -->
d17 1
a17 1
   similar hardware and all committed transaction will remain intact.
@


1.31
log
@Replace "--" and "---" with "&mdash;" as appropriate, for better-looking
output.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.30 2004/11/15 04:35:57 neilc Exp $ -->
d3 2
a4 2
<chapter id="wal">
 <title>Write-Ahead Logging (<acronym>WAL</acronym>)</title>
d6 75
a80 23
 <indexterm zone="wal">
  <primary>WAL</primary>
 </indexterm>

 <indexterm>
  <primary>transaction log</primary>
  <see>WAL</see>
 </indexterm>

  <para>
   <firstterm>Write-Ahead Logging</firstterm> (<acronym>WAL</acronym>)
   is a standard approach to transaction logging.  Its detailed
   description may be found in most (if not all) books about
   transaction processing. Briefly, <acronym>WAL</acronym>'s central
   concept is that changes to data files (where tables and indexes
   reside) must be written only after those changes have been logged,
   that is, when log records describing the changes have been flushed
   to permanent storage. If we follow this procedure, we do not need
   to flush data pages to disk on every transaction commit, because we
   know that in the event of a crash we will be able to recover the
   database using the log: any changes that have not been applied to
   the data pages can be redone from the log records.  (This is
   roll-forward recovery, also known as REDO.)
d83 29
d322 1
a322 1
  <title>Internals</title>
@


1.30
log
@Fix typo, other minor improvements.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/wal.sgml,v 1.29 2004/08/08 04:34:43 tgl Exp $ -->
d90 1
a90 1
    of the database state --- if it is made over some period of time,
d309 1
a309 1
   segments in reverse order -- newest to oldest -- in order to find the
@


1.29
log
@Update admin guide's discussion of WAL to match present reality.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/wal.sgml,v 1.28 2004/03/09 16:57:47 neilc Exp $ -->
d107 11
a117 11
   <firstterm>Checkpoints</firstterm><indexterm><primary>checkpoint</></> are points in the sequence of
   transactions at which it is guaranteed that the data files have
   been updated with all information logged before the checkpoint.  At
   checkpoint time, all dirty data pages are flushed to disk and a
   special checkpoint record is written to the log file. As result, in
   the event of a crash, the recoverer knows from what point in the
   log (known as the redo record) it should start the REDO operation,
   since any changes made to data files before that record are already
   on disk. After a checkpoint has been made, any log segments written
   before the redo record are no longer needed and can be recycled or
   removed. (When <acronym>WAL</acronym> archiving is being done, the
@


1.28
log
@Refer to GUC variables using <xref> tags rather than <varname> tags,
where appropriate. Add "id" and "xreflabel" tags to the descriptions
of the GUC variables to facilitate this. Also make a few minor docs
cleanups.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/wal.sgml,v 1.27 2004/01/06 17:26:23 neilc Exp $ -->
d27 2
a28 4
   the data pages will first be redone from the log records (this is
   roll-forward recovery, also known as REDO) and then changes made by
   uncommitted transactions will be removed from the data pages
   (roll-backward recovery, UNDO).
d31 1
a31 1
  <sect1 id="wal-benefits-now">
d34 1
a34 1
   <indexterm zone="wal-benefits-now">
d39 2
a40 2
    The first obvious benefit of using <acronym>WAL</acronym> is a
    significantly reduced number of disk writes, since only the log
d42 3
a44 2
    commit; in multiuser environments, commits of many transactions
    may be accomplished with a single <function>fsync()</function> of
d47 3
a49 1
    flushing the data pages.
d75 1
a75 1
    fixed by additional <function>fsync()</function> calls, but it is
d77 1
a77 1
    <acronym>WAL</acronym>; <acronym>WAL</acronym> saves the entire data
a80 44
  </sect1>
  
  <sect1 id="wal-benefits-later">
   <title>Future Benefits</title>

   <para>
    The UNDO operation is not implemented. This means that changes
    made by aborted transactions will still occupy disk space and that
    a permanent <filename>pg_clog</filename> file to hold
    the status of transactions is still needed, since
    transaction identifiers cannot be reused. Once UNDO is implemented,
    <filename>pg_clog</filename> will no longer be required to be
    permanent; it will be possible to remove
    <filename>pg_clog</filename> at shutdown. (However, the urgency of
    this concern has decreased greatly with the adoption of a segmented
    storage method for <filename>pg_clog</filename>: it is no longer
    necessary to keep old <filename>pg_clog</filename> entries around
    forever.)
   </para>

   <para>
    With UNDO, it will also be possible to implement
    <firstterm>savepoints</firstterm><indexterm><primary>savepoint</></> to allow partial rollback of
    invalid transaction operations (parser errors caused by mistyping
    commands, insertion of duplicate primary/unique keys and so on)
    with the ability to continue or commit valid operations made by
    the transaction before the error.  At present, any error will
    invalidate the whole transaction and require a transaction abort.
   </para>

   <para>
    <acronym>WAL</acronym> offers the opportunity for a new method for
    database on-line backup and restore (<acronym>BAR</acronym>).  To
    use this method, one would have to make periodic saves of data
    files to another disk, a tape or another host and also archive the
    <acronym>WAL</acronym> log files.  The database file copy and the
    archived log files could be used to restore just as if one were
    restoring after a crash. Each time a new database file copy was
    made the old log files could be removed.  Implementing this
    facility will require the logging of data file and index creation
    and deletion; it will also require development of a method for
    copying the data files (operating system copy commands are not
    suitable).
   </para>
d83 10
a92 9
    A difficulty standing in the way of realizing these benefits is that
    they require saving <acronym>WAL</acronym> entries for considerable
    periods of time (e.g., as long as the longest possible transaction if
    transaction UNDO is wanted). The present <acronym>WAL</acronym>
    format is extremely bulky since it includes many disk page
    snapshots. This is not a serious concern at present, since the
    entries only need to be kept for one or two checkpoint intervals;
    but to achieve these future benefits some sort of compressed
    <acronym>WAL</acronym> format will be needed.
d102 2
a103 2
   Consult <xref linkend="runtime-config"> for details about setting
   configuration parameters.
d112 1
a112 1
   the event of a crash, the recoverer knows from what record in the
d116 3
a118 4
   before the redo records are no longer needed and can be recycled or
   removed. (When <acronym>WAL</acronym>-based <acronym>BAR</acronym> is
   implemented, the log segments would be archived before being recycled
   or removed.)
d122 2
a123 2
   The server spawns a special process every so often to create the
   next checkpoint.  A checkpoint is created every <xref
d140 1
a140 1
   output to the log, partially negating the goal of using a smaller
d145 16
a160 1
   There will be at least one 16 MB segment file, and will normally
d162 3
a164 1
   files.  You can use this to estimate space requirements for <acronym>WAL</acronym>.
d191 2
a192 2
   not occur often enough to prevent <acronym>WAL</acronym> buffers
   being written by <function>LogInsert</function>. On such systems
d195 5
a199 13
   linkend="guc-wal-buffers">.  The default number of <acronym>
   WAL</acronym> buffers is 8.  Increasing this value will
   correspondingly increase shared memory usage.
  </para>

  <para>
   Checkpoints are fairly expensive because they force all dirty kernel
   buffers to disk using the operating system <literal>sync()</> call.
   Busy servers may fill checkpoint segment files too quickly,
   causing excessive checkpointing. If such forced checkpoints happen
   more frequently than <xref linkend="guc-checkpoint-warning"> seconds, 
   a message, will be output to the server logs recommending increasing 
   <varname>checkpoint_segments</varname>.
d245 2
a246 2
   required from the administrator except ensuring that the additional
   disk-space requirements of the <acronym>WAL</acronym> logs are met,
d254 2
a255 2
   segment files, each 16 MB in size.  Each segment is divided into 8
   kB pages. The log record headers are described in
d259 2
a260 2
   <filename>0000000000000000</filename>.  The numbers do not wrap, at
   present, but it should take a very long time to exhaust the
d284 3
a286 2
   disk drives<indexterm><primary>disk drive</></> that falsely report a successful write to the kernel,
   when, in fact, they have only cached the data and not yet stored it
d307 10
a316 6
   Using <filename>pg_control</filename> to get the checkpoint
   position speeds up the recovery process, but to handle possible
   corruption of <filename>pg_control</filename>, we should actually
   implement the reading of existing log segments in reverse order --
   newest to oldest -- in order to find the last checkpoint.  This has
   not been implemented, yet.
@


1.27
log
@Make the 'wal_debug' GUC variable a boolean (rather than an integer), and
hide it behind #ifdef WAL_DEBUG blocks.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/wal.sgml,v 1.26 2003/11/29 19:51:38 pgsql Exp $ -->
d165 4
a168 4
   The server spawns a special process every so often
   to create the next checkpoint.  A checkpoint is created every
   <varname>checkpoint_segments</varname> log segments, or every
   <varname>checkpoint_timeout</varname> seconds, whichever comes first.
d207 5
a211 5
   is used on every database low level modification (for example,
   row insertion) at a time when an exclusive lock is held on
   affected data pages, so the operation needs to be as fast as
   possible.  What is worse, writing <acronym>WAL</acronym> buffers may
   also force the creation of a new log segment, which takes even more
d220 3
a222 3
   modifying the configuration parameter <varname>wal_buffers</varname>.
   The default number of <acronym>
   WAL</acronym> buffers is 8.  Increasing this value will 
d231 1
a231 1
   more frequently than <varname>checkpoint_warning</varname> seconds, 
d237 1
a237 1
   The <varname>commit_delay</varname> parameter defines for how many
d243 2
a244 2
   <varname>fsync</varname>
   is not enabled, nor if fewer than <varname>commit_siblings</varname>
d255 1
a255 1
   The <varname>wal_sync_method</varname> parameter determines how
d265 6
a270 5
   Enabling the <varname>wal_debug</varname> configuration parameter
   will result in each <function>LogInsert</function> and
   <function>LogFlush</function> <acronym>WAL</acronym> call being
   logged to the server log. This option may be replaced by a more
   general mechanism in the future.
@


1.26
log
@
$Header: -> $PostgreSQL Changes ...
@
text
@d1 1
a1 1
<!-- $PostgreSQL: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.25 2003/09/20 20:12:05 tgl Exp $ -->
d22 4
a25 4
   that is, when log records have been flushed to permanent
   storage. If we follow this procedure, we do not need to flush
   data pages to disk on every transaction commit, because we know
   that in the event of a crash we will be able to recover the
d190 1
a190 1
   files.  You can use this to estimate space requirements for WAL.
d257 1
a257 1
    WAL updates out to disk. 
d265 2
a266 2
   Setting the <varname>wal_debug</varname> parameter to any nonzero
   value will result in each <function>LogInsert</function> and
d268 1
a268 2
   logged to the server log.  At present, it makes no difference what
   the nonzero value is.  This option may be replaced by a more
@


1.25
log
@Fix a batch of speling misteaks identified by Peter's spell-checker tool.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.24 2003/08/31 17:32:20 petere Exp $ -->
@


1.24
log
@Add/edit index entries.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.23 2003/03/24 14:32:51 petere Exp $ -->
d242 4
a245 3
   of them flushed with a single log sync. No sleep will occur if <varname>fsync</varname>
   is not enabled or if fewer than <varname>commit_siblings</varname>
   other sessons are currently in active transactions; this avoids
@


1.23
log
@Another big editing pass for consistent content and presentation.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.22 2002/11/15 02:44:54 momjian Exp $ -->
d6 9
d36 4
d102 1
a102 1
    <firstterm>savepoints</firstterm> to allow partial rollback of
d149 1
a149 1
   <firstterm>Checkpoints</firstterm> are points in the sequence of
d317 1
a317 1
   disk drives that falsely report a successful write to the kernel,
@


1.23.2.1
log
@Update this branch to match CVS head, includes WIN32 improvements.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.24 2003/08/31 17:32:20 petere Exp $ -->
a5 9
 <indexterm zone="wal">
  <primary>WAL</primary>
 </indexterm>

 <indexterm>
  <primary>transaction log</primary>
  <see>WAL</see>
 </indexterm>

a26 4
   <indexterm zone="wal-benefits-now">
    <primary>fsync</primary>
   </indexterm>

d89 1
a89 1
    <firstterm>savepoints</firstterm><indexterm><primary>savepoint</></> to allow partial rollback of
d136 1
a136 1
   <firstterm>Checkpoints</firstterm><indexterm><primary>checkpoint</></> are points in the sequence of
d304 1
a304 1
   disk drives<indexterm><primary>disk drive</></> that falsely report a successful write to the kernel,
@


1.22
log
@Add checkpoint_warning to warn of excessive checkpoints caused by too
few WAL files.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.21 2002/11/02 22:23:01 tgl Exp $ -->
a5 2
 <note>
  <title>Author</title>
d7 1
a7 9
   Vadim Mikheev and Oliver Elphick
  </para>
 </note>

 <sect1 id="wal-general">
  <title>General Description</Title>

  <para>
   <firstterm>Write Ahead Logging</firstterm> (<acronym>WAL</acronym>)
d12 1
a12 1
   reside) must be written only after those changes have been logged -
d14 1
a14 1
   storage. When we follow this procedure, we do not need to flush
d21 1
a21 1
   (roll-backward recovery - UNDO).
d24 2
a25 2
  <sect2 id="wal-benefits-now">
   <title>Immediate Benefits of <acronym>WAL</acronym></title>
d47 1
a47 1
      <simpara>index tuples pointing to nonexistent table rows</simpara>
d51 1
a51 1
      <simpara>index tuples lost in split operations</simpara>
d67 1
a67 1
  </sect2>
d69 1
a69 1
  <sect2 id="wal-benefits-later">
d73 1
a73 1
    UNDO operation is not implemented. This means that changes
d75 3
a77 3
    we still need a permanent <filename>pg_clog</filename> file to hold
    the status of transactions, since we are not able to re-use
    transaction identifiers. Once UNDO is implemented,
d82 1
a82 1
    storage method for <filename>pg_clog</filename> --- it is no longer
d115 1
a115 1
    periods of time (eg, as long as the longest possible transaction if
d123 1
a123 81
  </sect2>
 </sect1>

 <sect1 id="wal-implementation">
  <title>Implementation</title>

  <para>
   <acronym>WAL</acronym> is automatically enabled from release 7.1
   onwards. No action is required from the administrator with the
   exception of ensuring that the additional disk-space requirements
   of the <acronym>WAL</acronym> logs are met, and that any necessary
   tuning is done (see <xref linkend="wal-configuration">).
  </para>

  <para>
   <acronym>WAL</acronym> logs are stored in the directory
   <Filename><replaceable>$PGDATA</replaceable>/pg_xlog</Filename>, as
   a set of segment files, each 16 MB in size.  Each segment is
   divided into 8 kB pages. The log record headers are described in
   <filename>access/xlog.h</filename>; record content is dependent on
   the type of event that is being logged.  Segment files are given
   ever-increasing numbers as names, starting at
   <filename>0000000000000000</filename>.  The numbers do not wrap, at
   present, but it should take a very long time to exhaust the
   available stock of numbers.
  </para>

  <para>
   The <acronym>WAL</acronym> buffers and control structure are in
   shared memory, and are handled by the backends; they are protected
   by lightweight locks.  The demand on shared memory is dependent on the
   number of buffers.  The default size of the <acronym>WAL</acronym>
   buffers is 8 buffers of 8 kB each, or 64 kB total.
  </para>

  <para>
   It is of advantage if the log is located on another disk than the
   main database files.  This may be achieved by moving the directory,
   <filename>pg_xlog</filename>, to another location (while the
   postmaster is shut down, of course) and creating a symbolic link
   from the original location in <replaceable>$PGDATA</replaceable> to
   the new location.
  </para>

  <para>
   The aim of <acronym>WAL</acronym>, to ensure that the log is
   written before database records are altered, may be subverted by
   disk drives that falsely report a successful write to the kernel,
   when, in fact, they have only cached the data and not yet stored it
   on the disk.  A power failure in such a situation may still lead to
   irrecoverable data corruption.  Administrators should try to ensure
   that disks holding <productname>PostgreSQL</productname>'s
   log files do not make such false reports.
  </para>

  <sect2 id="wal-recovery">
   <title>Database Recovery with <acronym>WAL</acronym></title>

   <para>
    After a checkpoint has been made and the log flushed, the
    checkpoint's position is saved in the file
    <filename>pg_control</filename>. Therefore, when recovery is to be
    done, the backend first reads <filename>pg_control</filename> and
    then the checkpoint record; then it performs the REDO operation by
    scanning forward from the log position indicated in the checkpoint
    record.
    Because the entire content of data pages is saved in the log on the
    first page modification after a checkpoint, all pages changed since
    the checkpoint will be restored to a consistent state.
   </para>

   <para>
    Using <filename>pg_control</filename> to get the checkpoint
    position speeds up the recovery process, but to handle possible
    corruption of <filename>pg_control</filename>, we should actually
    implement the reading of existing log segments in reverse order --
    newest to oldest -- in order to find the last checkpoint.  This has
    not been implemented, yet.
   </para>
  </sect2>
 </sect1>
d129 1
a129 1
   There are several <acronym>WAL</acronym>-related parameters that
d145 1
a145 1
   before the undo records are no longer needed and can be recycled or
d152 1
a152 1
   The postmaster spawns a special backend process every so often
d154 2
a155 2
   <varname>CHECKPOINT_SEGMENTS</varname> log segments, or every
   <varname>CHECKPOINT_TIMEOUT</varname> seconds, whichever comes first.
d162 2
a163 2
   Reducing <varname>CHECKPOINT_SEGMENTS</varname> and/or
   <varname>CHECKPOINT_TIMEOUT</varname> causes checkpoints to be done
d175 9
a183 9
   There will be at least one 16MB segment file, and will normally 
   not be more than 2 * <varname>CHECKPOINT_SEGMENTS</varname>
   + 1 files.  You can use this to estimate space requirements for 
   WAL.  Ordinarily, when old log segment files are no longer needed, 
   they are recycled (renamed to become the next sequential future 
   segments). If, due to a short-term peak of log output rate, there 
   are more than 2 * <varname>CHECKPOINT_SEGMENTS</varname> + 1 segment files, 
   the unneeded segment files will be deleted instead of recycled until the
   system gets back under this limit.
d195 1
a195 1
   tuple insertion) at a time when an exclusive lock is held on
d207 2
a208 2
   modifying the <filename>postgresql.conf</filename> <varname>
   WAL_BUFFERS</varname> parameter. The default number of <acronym>
d218 1
a218 1
   more frequently than <varname>CHECKPOINT_WARNING</varname> seconds, 
d220 1
a220 1
   <varname>CHECKPOINT_SEGMENTS</varname>.
d224 2
a225 2
   The <varname>COMMIT_DELAY</varname> parameter defines for how many
   microseconds the backend will sleep after writing a commit
d228 1
a228 1
   backends to add their commit records to the log so as to have all
d230 3
a232 3
   is not enabled or if fewer than <varname>COMMIT_SIBLINGS</varname>
   other backends are not currently in active transactions; this avoids
   sleeping when it's unlikely that any other backend will commit soon.
d234 2
a235 2
   ten milliseconds, so that any nonzero <varname>COMMIT_DELAY</varname>
   setting between 1 and 10000 microseconds will have the same effect.
d241 1
a241 1
   The <varname>WAL_SYNC_METHOD</varname> parameter determines how
d246 1
a246 1
   Note that this parameter is irrelevant if <varname>FSYNC</varname>
d251 1
a251 1
   Setting the <varname>WAL_DEBUG</varname> parameter to any nonzero
d254 1
a254 1
   logged to standard error.  At present, it makes no difference what
d257 75
@


1.21
log
@Fix some misstatements in WAL parameter discussion.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.20 2002/09/06 20:26:00 momjian Exp $ -->
d301 10
@


1.20
log
@Fix another typo.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.19 2002/09/06 20:08:06 momjian Exp $ -->
a225 27
   There are two commonly used <acronym>WAL</acronym> functions:
   <function>LogInsert</function> and <function>LogFlush</function>.
   <function>LogInsert</function> is used to place a new record into
   the <acronym>WAL</acronym> buffers in shared memory. If there is no
   space for the new record, <function>LogInsert</function> will have
   to write (move to kernel cache) a few filled <acronym>WAL</acronym>
   buffers. This is undesirable because <function>LogInsert</function>
   is used on every database low level modification (for example,
   tuple insertion) at a time when an exclusive lock is held on
   affected data pages, so the operation needs to be as fast as
   possible.  What is worse, writing <acronym>WAL</acronym> buffers may
   also force the creation of a new log segment, which takes even more
   time. Normally, <acronym>WAL</acronym> buffers should be written
   and flushed by a <function>LogFlush</function> request, which is
   made, for the most part, at transaction commit time to ensure that
   transaction records are flushed to permanent storage. On systems
   with high log output, <function>LogFlush</function> requests may
   not occur often enough to prevent <acronym>WAL</acronym> buffers
   being written by <function>LogInsert</function>. On such systems
   one should increase the number of <acronym>WAL</acronym> buffers by
   modifying the <filename>postgresql.conf</filename> <varname>
   WAL_BUFFERS</varname> parameter. The default number of <acronym>
   WAL</acronym> buffers is 8.  Increasing this value will 
   correspondingly increase shared memory usage.
  </para>

  <para>
a241 13
   The checkpoint maker is also able to create a few log segments for
   future use, so as to avoid the need for
   <function>LogInsert</function> or <function>LogFlush</function> to
   spend time in creating them.  (If that happens, the entire database
   system will be delayed by the creation operation, so it's better if
   the files can be created in the checkpoint maker, which is not on
   anyone's critical path.)
   By default a new 16MB segment file is created only if more than 75% of
   the current segment has been used.  This is inadequate if the system
   generates more than 4MB of log output between checkpoints.
  </para>

  <para>
d266 1
a266 1
   not be more than <varname>CHECKPOINT_SEGMENTS</varname>
d271 1
a271 1
   are more than <varname>CHECKPOINT_SEGMENTS</varname> + 1 segment files, 
d274 27
@


1.19
log
@Fix typo.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.18 2002/08/30 16:53:34 momjian Exp $ -->
d306 1
a306 1
   not be more than <varname>CHECKPOINT_SEGMENTS</varname>)
@


1.18
log
@Improve wording after wal_files removal.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.17 2002/08/30 16:50:50 momjian Exp $ -->
d311 1
a311 1
   are more than <varname>CHECKPOINT_SEGMENTS</varname>) + 1 segment files, 
@


1.17
log
@Complete TODO item:

	* Remove wal_files postgresql.conf option because WAL files are
	  now recycled
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/wal.sgml,v 1.16 2002/07/05 19:06:11 momjian Exp $ -->
d305 4
a308 4
   The number of 16MB segment files will always be at least
   1, and will normally not exceed <varname>CHECKPOINT_SEGMENTS</varname>)
   + 1.  This may be used to estimate space requirements for WAL. 
   Ordinarily, when old log segment files are no longer needed, 
@


1.16
log
@Don't document that UNDO is certain to be added in the future.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.15 2002/02/11 23:25:14 momjian Exp $ -->
a278 3
   One can instruct the server to pre-create up to 64 log segments
   at checkpoint time by modifying the <varname>WAL_FILES</varname>
   configuration parameter.
d306 8
a313 14
   <varname>WAL_FILES</varname> + 1, and will normally not exceed
   <varname>WAL_FILES</varname> + MAX(<varname>WAL_FILES</varname>,
   <varname>CHECKPOINT_SEGMENTS</varname>) + 1. This may be used to
   estimate space requirements for WAL. Ordinarily, when an old log
   segment files are no longer needed, they are recycled (renamed to
   become the next sequential future segments). If, due to a short-term
   peak of log output rate, there are more than
   <varname>WAL_FILES</varname> + MAX(<varname>WAL_FILES</varname>,
   <varname>CHECKPOINT_SEGMENTS</varname>) + 1 segment files, then
   unneeded segment files will be deleted instead of recycled until the
   system gets back under this limit. (If this happens on a regular
   basis, <varname>WAL_FILES</varname> should be increased to avoid it.
   Deleting log segments that will only have to be created again later
   is expensive and pointless.)
@


1.15
log
@Update wal files computation documentation.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.14 2001/11/28 20:49:10 petere Exp $ -->
d83 1
a83 2
    In this first release of <acronym>WAL</acronym>, UNDO operation is
    not implemented, because of lack of time. This means that changes
@


1.14
log
@proof-reading
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.13 2001/11/21 06:09:45 thomas Exp $ -->
d73 2
a74 2
    <acronym>WAL</acronym>; <acronym>WAL</acronym> saves the entire
    data page content in the log if that is required to ensure page
d88 1
a88 1
    transaction identifiers.  Once UNDO is implemented,
d91 2
a92 2
    <filename>pg_clog</filename> at shutdown.  (However, the urgency
    of this concern has decreased greatly with the adoption of a segmented
d124 9
a132 9
    A difficulty standing in the way of realizing these benefits is that they
    require saving <acronym>WAL</acronym> entries for considerable periods
    of time (eg, as long as the longest possible transaction if transaction
    UNDO is wanted).  The present <acronym>WAL</acronym> format is
    extremely bulky since it includes many disk page snapshots.
    This is not a serious concern at present, since the entries only need
    to be kept for one or two checkpoint intervals; but to achieve
    these future benefits some sort of compressed <acronym>WAL</acronym>
    format will be needed.
d247 4
a250 3
   modifying the <varname>WAL_BUFFERS</varname> parameter. The default
   number of <acronym>WAL</acronym> buffers is 8.  Increasing this
   value will correspondingly increase shared memory usage.
d297 9
a305 9
   <varname>CHECKPOINT_TIMEOUT</varname> causes checkpoints to be
   done more often.  This allows faster after-crash recovery (since
   less work will need to be redone).  However, one must balance this against
   the increased cost of flushing dirty data pages more often.  In addition,
   to ensure data page consistency, the first modification of a data page
   after each checkpoint results in logging the entire page content.
   Thus a smaller checkpoint interval increases the volume of output to
   the log, partially negating the goal of using a smaller interval, and
   in any case causing more disk I/O.
d311 13
a323 11
   <varname>WAL_FILES</varname> + 2 * <varname>CHECKPOINT_SEGMENTS</varname>
   + 1.  This may be used to estimate space requirements for WAL.  Ordinarily,
   when an old log segment file is no longer needed, it is recycled (renamed
   to become the next sequential future segment).  If, due to a short-term
   peak of log output rate, there are more than <varname>WAL_FILES</varname> +
   2 * <varname>CHECKPOINT_SEGMENTS</varname> + 1 segment files, then unneeded
   segment files will be deleted instead of recycled until the system gets
   back under this limit.  (If this happens on a regular basis,
   <varname>WAL_FILES</varname> should be increased to avoid it.  Deleting log
   segments that will only have to be created again later is expensive and
   pointless.)
@


1.13
log
@Deprecate 'current' date/time constant.
Purge "Postgres" in favor of "PostgreSQL" in docs. ref/ not yet done.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.12 2001/10/26 23:10:21 tgl Exp $ -->
d57 1
a57 1
      <simpara>index tuples pointing to non-existent table rows</simpara>
d151 2
a152 2
   a set of segment files, each 16MB in size.  Each segment is
   divided into 8KB pages. The log record headers are described in
d166 1
a166 1
   buffers is 8 8KB buffers, or 64KB.
d211 1
a211 1
    not yet been done in release 7.1.
d351 1
a351 1
   Setting the <varname>WAL_DEBUG</varname> parameter to any non-zero
d355 1
a355 1
   the non-zero value is.  This option may be replaced by a more
@


1.12
log
@Update WAL configuration discussion to reflect post-7.1 tweaking.
Minor copy-editing.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.11 2001/09/29 04:02:19 tgl Exp $ -->
d342 2
a343 1
   Postgres will ask the kernel to force WAL updates out to disk.
@


1.11
log
@Implement new 'lightweight lock manager' that's intermediate between
existing lock manager and spinlocks: it understands exclusive vs shared
lock but has few other fancy features.  Replace most uses of spinlocks
with lightweight locks.  All remaining uses of spinlocks have very short
lock hold times (a few dozen instructions), so tweak spinlock backoff
code to work efficiently given this assumption.  All per my proposal on
pghackers 26-Sep-01.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.10 2001/09/22 03:59:17 momjian Exp $ -->
d91 5
a95 2
    <filename>pg_clog</filename> at shutdown, split it into segments
    and remove old segments.
d122 12
d151 2
a152 2
   a set of segment files, each 16 MB in size.  Each segment is
   divided into 8 kB pages. The log record headers are described in
d165 2
a166 2
   number of buffers; the default size of the <acronym>WAL</acronym>
   buffers is 64 kB.
d184 2
a185 2
   irrecoverable data corruption; administrators should try to ensure
   that disks holding <productname>PostgreSQL</productname>'s data and
d197 6
a202 5
    then the checkpoint record; next it reads the redo record, whose
    position is saved in the checkpoint, and begins the REDO operation.
    Because the entire content of the pages is saved in the log on the
    first page modification after a checkpoint, the pages will be first
    restored to a consistent state.
d236 3
a238 3
   affected data pages and the operation is supposed to be as fast as
   possible; what is worse, writing <acronym>WAL</acronym> buffers may
   also cause the creation of a new log segment, which takes even more
d249 1
a249 1
   value will have an impact on shared memory usage.
d262 7
a268 4
   before the undo records are removed, so checkpoints are used to free
   disk space in the <acronym>WAL</acronym> directory. (When
   <acronym>WAL</acronym>-based <acronym>BAR</acronym> is implemented,
   the log segments can be archived instead of just being removed.)
d272 8
a279 8
   spend time in creating them.
  </para>

  <para>
   The <acronym>WAL</acronym> log is held on the disk as a set of 16
   MB files called <firstterm>segments</firstterm>.  By default a new
   segment is created only if more than 75% of the current segment is
   used. One can instruct the server to pre-create up to 64 log segments
a284 9
   For faster after-crash recovery, it would be better to create
   checkpoints more often.  However, one should balance this against
   the cost of flushing dirty data pages; in addition, to ensure data
   page consistency, the first modification of a data page after each
   checkpoint results in logging the entire page content, thus
   increasing output to log and the log's size.
  </para>

  <para>
d295 29
d336 2
@


1.10
log
@WAL file numbers:  Sequenatial -> ever-increasing
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.9 2001/09/09 23:52:12 petere Exp $ -->
d149 1
a149 1
   by spinlocks.  The demand on shared memory is dependent on the
@


1.9
log
@Markup additions and spell check. (covers Admin Guide)
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.8 2001/08/25 18:52:41 tgl Exp $ -->
d140 1
a140 1
   sequential numbers as names, starting at
@


1.8
log
@Replace implementation of pg_log as a relation accessed through the
buffer manager with 'pg_clog', a specialized access method modeled
on pg_xlog.  This simplifies startup (don't need to play games to
open pg_log; among other things, OverrideTransactionSystem goes away),
should improve performance a little, and opens the door to recycling
commit log space by removing no-longer-needed segments of the commit
log.  Actual recycling is not there yet, but I felt I should commit
this part separately since it'd still be useful if we chose not to
do transaction ID wraparound.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.7 2001/05/17 21:50:16 petere Exp $ -->
d41 1
a41 1
    commit; in multi-user environments, commits of many transactions
d290 1
a290 1
   of them flushed with a single log sync. No sleep will occur if fsync
@


1.7
log
@Change "indices" to "indexes", per OED.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.6 2001/05/15 13:57:37 momjian Exp $ -->
d86 1
a86 1
    we still need a permanent <filename>pg_log</filename> file to hold
d89 1
a89 1
    <filename>pg_log</filename> will no longer be required to be
d91 1
a91 1
    <filename>pg_log</filename> at shutdown, split it into segments
@


1.6
log
@redo record -> undo record, Tom Lane.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.5 2001/03/16 05:44:33 tgl Exp $ -->
d21 1
a21 1
   concept is that changes to data files (where tables and indices
d70 1
a70 1
    Problems with indices (problems 1 and 2) could possibly have been
@


1.5
log
@Support syncing WAL log to disk using either fsync(), fdatasync(),
O_SYNC, or O_DSYNC (as available on a given platform).  Add GUC parameter
to control sync method.
Also, add defense to XLogWrite to prevent it from going nuts if passed
a target write position that's past the end of the buffers so far filled
by XLogInsert.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.4 2001/03/13 01:17:05 tgl Exp $ -->
d246 1
a246 1
   before the redo record are removed, so checkpoints are used to free
@


1.4
log
@XLOG (and related) changes:

* Store two past checkpoint locations, not just one, in pg_control.
  On startup, we fall back to the older checkpoint if the newer one
  is unreadable.  Also, a physical copy of the newest checkpoint record
  is kept in pg_control for possible use in disaster recovery (ie,
  complete loss of pg_xlog).  Also add a version number for pg_control
  itself.  Remove archdir from pg_control; it ought to be a GUC
  parameter, not a special case (not that it's implemented yet anyway).

* Suppress successive checkpoint records when nothing has been entered
  in the WAL log since the last one.  This is not so much to avoid I/O
  as to make it actually useful to keep track of the last two
  checkpoints.  If the things are right next to each other then there's
  not a lot of redundancy gained...

* Change CRC scheme to a true 64-bit CRC, not a pair of 32-bit CRCs
  on alternate bytes.  Polynomial borrowed from ECMA DLT1 standard.

* Fix XLOG record length handling so that it will work at BLCKSZ = 32k.

* Change XID allocation to work more like OID allocation.  (This is of
  dubious necessity, but I think it's a good idea anyway.)

* Fix a number of minor bugs, such as off-by-one logic for XLOG file
  wraparound at the 4 gig mark.

* Add documentation and clean up some coding infelicities; move file
  format declarations out to include files where planned contrib
  utilities can get at them.

* Checkpoint will now occur every CHECKPOINT_SEGMENTS log segments or
  every CHECKPOINT_TIMEOUT seconds, whichever comes first.  It is also
  possible to force a checkpoint by sending SIGUSR1 to the postmaster
  (undocumented feature...)

* Defend against kill -9 postmaster by storing shmem block's key and ID
  in postmaster.pid lockfile, and checking at startup to ensure that no
  processes are still connected to old shmem block (if it still exists).

* Switch backends to accept SIGQUIT rather than SIGUSR1 for emergency
  stop, for symmetry with postmaster and xlog utilities.  Clean up signal
  handling in bootstrap.c so that xlog utilities launched by postmaster
  will react to signals better.

* Standalone bootstrap now grabs lockfile in target directory, as added
  insurance against running it in parallel with live postmaster.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.3 2001/02/26 00:50:07 tgl Exp $ -->
a284 9
   Setting the <varname>WAL_DEBUG</varname> parameter to any non-zero
   value will result in each <function>LogInsert</function> and
   <function>LogFlush</function> <acronym>WAL</acronym> call being
   logged to standard error.  At present, it makes no difference what
   the non-zero value is.  This option may be replaced by a more
   general mechanism in the future.
  </para>

  <para>
d297 18
@


1.3
log
@Implement COMMIT_SIBLINGS parameter to allow pre-commit delay to occur
only if at least N other backends currently have open transactions.  This
is not a great deal of intelligence about whether a delay might be
profitable ... but it beats no intelligence at all.  Note that the default
COMMIT_DELAY is still zero --- this new code does nothing unless that
setting is changed.
Also, mark ENABLEFSYNC as a system-wide setting.  It's no longer safe to
allow that to be set per-backend, since we may be relying on some other
backend's fsync to have synced the WAL log.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.2 2001/02/18 04:50:43 tgl Exp $ -->
d260 1
a260 1
   used. One can instruct the server to create up to 64 log segments
d275 6
a280 5
   By default, the postmaster spawns a special backend process to
   create the next checkpoint 300 seconds after the previous
   checkpoint's creation.  One can change this interval by modifying
   the <varname>CHECKPOINT_TIMEOUT</varname> parameter.  It is also
   possible to force a checkpoint by using the SQL command
@


1.2
log
@Change default commit_delay to zero, update documentation.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/wal.sgml,v 1.1 2001/01/24 23:15:19 petere Exp $ -->
d298 7
a304 4
   of them flushed with a single log sync. Unfortunately, this
   mechanism is not fully implemented at release 7.1, so there is at
   present usually no benefit to be gained from increasing this parameter
   above its default value of zero.
@


1.1
log
@WAL documentation, from Oliver Elphick and Vadim Mikheev.
@
text
@d1 1
a1 1
<!-- $Header$ -->
d293 3
a295 3
   The <varname>COMMIT_DELAY</varname> parameter defines for how long
   the backend will be forced to sleep after writing a commit record
   to the log with <function>LogInsert</function> call but before
d300 2
a301 2
   present no point in changing this parameter from its default value
   of 5 microseconds.
@
