head	1.68;
access;
symbols
	REL9_0_0:1.67.2.1
	REL9_1_ALPHA1:1.68
	REL9_0_RC1:1.67.2.1
	REL9_0_BETA4:1.67
	REL9_0_STABLE:1.67.0.2
	REL9_0_BETA3:1.67
	REL9_0_BETA2:1.67
	REL7_4_29:1.35.2.1
	REL8_0_25:1.45
	REL8_1_21:1.49
	REL8_2_17:1.55
	REL8_3_11:1.58
	REL8_4_4:1.63
	REL9_0_BETA1:1.66
	REL9_0_ALPHA5_BRANCH:1.65.0.4
	REL9_0_ALPHA5:1.65
	REL7_4_28:1.35.2.1
	REL8_0_24:1.45
	REL8_1_20:1.49
	REL8_2_16:1.55
	REL8_3_10:1.58
	REL8_4_3:1.63
	REL9_0_ALPHA4:1.65
	REL9_0_ALPHA4_BRANCH:1.65.0.2
	REL8_5_ALPHA3:1.64
	REL8_5_ALPHA3_BRANCH:1.64.0.6
	REL7_4_27:1.35.2.1
	REL8_0_23:1.45
	REL8_1_19:1.49
	REL8_2_15:1.55
	REL8_3_9:1.58
	REL8_4_2:1.63
	REL8_5_ALPHA2:1.64
	REL8_5_ALPHA2_BRANCH:1.64.0.4
	REL7_4_26:1.35.2.1
	REL8_0_22:1.45
	REL8_1_18:1.49
	REL8_2_14:1.55
	REL8_3_8:1.58
	REL8_4_1:1.63
	REL8_5_ALPHA1:1.64
	REL8_5_ALPHA1_BRANCH:1.64.0.2
	REL8_4_STABLE:1.63.0.2
	REL8_4_0:1.63
	REL8_4_RC2:1.63
	REL8_4_RC1:1.63
	REL8_4_BETA2:1.63
	REL8_4_BETA1:1.62
	REL7_4_25:1.35.2.1
	REL8_0_21:1.45
	REL8_1_17:1.49
	REL8_2_13:1.55
	REL8_3_7:1.58
	REL7_4_24:1.35.2.1
	REL8_0_20:1.45
	REL8_1_16:1.49
	REL8_2_12:1.55
	REL8_3_6:1.58
	REL7_4_23:1.35.2.1
	REL8_0_19:1.45
	REL8_1_15:1.49
	REL8_2_11:1.55
	REL8_3_5:1.58
	REL7_4_22:1.35.2.1
	REL8_0_18:1.45
	REL8_1_14:1.49
	REL8_2_10:1.55
	REL8_3_4:1.58
	REL7_4_21:1.35.2.1
	REL8_0_17:1.45
	REL8_1_13:1.49
	REL8_2_9:1.55
	REL8_3_3:1.58
	REL7_4_20:1.35.2.1
	REL8_0_16:1.45
	REL8_1_12:1.49
	REL8_2_8:1.55
	REL8_3_2:1.58
	REL8_2_7:1.55
	REL8_3_1:1.58
	REL8_3_STABLE:1.58.0.2
	REL8_3_0:1.58
	REL8_3_RC2:1.58
	REL7_3_21:1.29
	REL7_4_19:1.35.2.1
	REL8_0_15:1.45
	REL8_1_11:1.49
	REL8_2_6:1.55
	REL8_3_RC1:1.58
	REL8_3_BETA4:1.58
	REL8_3_BETA3:1.58
	REL8_3_BETA2:1.58
	REL8_3_BETA1:1.58
	REL7_3_20:1.29
	REL7_4_18:1.35.2.1
	REL8_0_14:1.45
	REL8_1_10:1.49
	REL8_2_5:1.55
	REL7_3_19:1.29
	REL7_4_17:1.35.2.1
	REL8_0_13:1.45
	REL8_1_9:1.49
	REL8_2_4:1.55
	REL8_0_12:1.45
	REL8_1_8:1.49
	REL8_2_3:1.55
	REL7_3_18:1.29
	REL7_4_16:1.35.2.1
	REL8_0_11:1.45
	REL8_1_7:1.49
	REL8_2_2:1.55
	REL8_0_10:1.45
	REL8_1_6:1.49
	REL8_2_1:1.55
	REL7_4_15:1.35.2.1
	REL7_3_17:1.29
	REL8_2_STABLE:1.55.0.2
	REL8_2_0:1.55
	REL8_2_RC1:1.55
	REL8_2_BETA3:1.55
	REL8_2_BETA2:1.55
	REL8_1_5:1.49
	REL8_0_9:1.45
	REL7_4_14:1.35.2.1
	REL7_3_16:1.29
	REL8_2_BETA1:1.55
	REL7_3_15:1.29
	REL7_4_13:1.35.2.1
	REL8_0_8:1.45
	REL8_1_4:1.49
	REL7_3_14:1.29
	REL7_4_12:1.35.2.1
	REL8_0_7:1.45
	REL8_1_3:1.49
	REL7_3_13:1.29
	REL7_4_11:1.35.2.1
	REL8_0_6:1.45
	REL8_1_2:1.49
	REL7_3_12:1.29
	REL7_4_10:1.35.2.1
	REL8_0_5:1.45
	REL8_1_1:1.49
	REL8_1_STABLE:1.49.0.2
	REL8_1_0:1.49
	REL8_1_0RC1:1.49
	REL8_1_0BETA4:1.49
	REL8_1_0BETA3:1.47
	REL7_3_11:1.29
	REL7_4_9:1.35.2.1
	REL8_0_4:1.45
	REL8_1_0BETA2:1.47
	REL8_1_0BETA1:1.47
	REL7_2_8:1.25
	REL7_3_10:1.29
	REL7_4_8:1.35.2.1
	REL8_0_3:1.45
	REL8_0_2:1.45
	REL7_2_7:1.25
	REL7_3_9:1.29
	REL7_4_7:1.35.2.1
	REL8_0_1:1.45
	REL8_0_STABLE:1.45.0.4
	REL8_0_0:1.45.0.2
	REL8_0_0RC5:1.45
	REL8_0_0RC4:1.45
	REL8_0_0RC3:1.45
	REL8_0_0RC2:1.44
	REL8_0_0RC1:1.44
	REL8_0_0BETA5:1.42
	REL8_0_0BETA4:1.41
	REL7_4_6:1.35.2.1
	REL7_3_8:1.29
	REL7_2_6:1.25
	REL8_0_0BETA3:1.41
	REL8_0_0BETA2:1.41
	REL7_2_5:1.25
	REL7_4_5:1.35.2.1
	REL7_3_7:1.29
	REL7_4_4:1.35.2.1
	REL8_0_0BETA1:1.40
	REL7_4_3:1.35.2.1
	REL7_4_2:1.35.2.1
	REL7_3_6:1.29
	REL7_4_1:1.35.2.1
	REL7_3_5:1.29
	REL7_4:1.35.2.1
	REL7_4_RC2:1.35.2.1
	REL7_4_STABLE:1.35.0.2
	REL7_4_RC1:1.35
	REL7_4_BETA5:1.34
	REL7_4_BETA4:1.34
	REL7_4_BETA3:1.34
	REL7_4_BETA2:1.32
	WIN32_DEV:1.32.0.2
	REL7_4_BETA1:1.32
	REL7_3_4:1.29
	REL7_3_2:1.29
	REL7_2_4:1.25
	REL7_3_STABLE:1.29.0.2
	REL7_2_3:1.25
	REL7_2_STABLE:1.25.0.2
	REL7_2:1.25
	REL7_2_RC2:1.25
	REL7_2_RC1:1.25
	REL7_2_BETA5:1.24
	REL7_2_BETA4:1.23
	REL7_2_BETA3:1.21
	REL7_2_BETA2:1.21
	REL7_2_BETA1:1.21
	REL7_1_2:1.17
	REL7_1_STABLE:1.17.0.2
	REL7_1_BETA:1.14
	REL7_1_BETA3:1.16
	REL7_1_BETA2:1.16
	REL7_1:1.17
	REL7_0_PATCHES:1.9.0.2
	REL7_0:1.9
	REL6_5_PATCHES:1.4.0.2
	REL6_5:1.4
	REL6_4:1.2.0.2
	release-6-3:1.1;
locks; strict;
comment	@# @;


1.68
date	2010.08.17.04.37.21;	author petere;	state Exp;
branches;
next	1.67;

1.67
date	2010.05.02.12.22.40;	author sriggs;	state Exp;
branches
	1.67.2.1;
next	1.66;

1.66
date	2010.04.03.07.22.55;	author petere;	state Exp;
branches;
next	1.65;

1.65
date	2010.02.03.17.25.06;	author momjian;	state Exp;
branches;
next	1.64;

1.64
date	2009.08.07.20.50.21;	author petere;	state Exp;
branches;
next	1.63;

1.63
date	2009.04.27.16.27.36;	author momjian;	state Exp;
branches;
next	1.62;

1.62
date	2009.02.12.13.26.03;	author petere;	state Exp;
branches;
next	1.61;

1.61
date	2009.02.11.14.03.41;	author petere;	state Exp;
branches;
next	1.60;

1.60
date	2008.09.05.12.11.18;	author petere;	state Exp;
branches;
next	1.59;

1.59
date	2008.05.30.00.04.32;	author tgl;	state Exp;
branches;
next	1.58;

1.58
date	2007.06.12.17.49.12;	author mha;	state Exp;
branches;
next	1.57;

1.57
date	2007.02.01.00.28.17;	author momjian;	state Exp;
branches;
next	1.56;

1.56
date	2007.01.31.20.56.18;	author momjian;	state Exp;
branches;
next	1.55;

1.55
date	2006.09.16.00.30.15;	author momjian;	state Exp;
branches;
next	1.54;

1.54
date	2006.07.19.17.02.59;	author tgl;	state Exp;
branches;
next	1.53;

1.53
date	2006.07.19.02.37.00;	author tgl;	state Exp;
branches;
next	1.52;

1.52
date	2006.06.18.15.38.36;	author petere;	state Exp;
branches;
next	1.51;

1.51
date	2006.04.06.18.54.36;	author petere;	state Exp;
branches;
next	1.50;

1.50
date	2006.01.22.05.20.32;	author neilc;	state Exp;
branches;
next	1.49;

1.49
date	2005.10.18.21.43.33;	author tgl;	state Exp;
branches;
next	1.48;

1.48
date	2005.10.13.23.41.07;	author tgl;	state Exp;
branches;
next	1.47;

1.47
date	2005.07.24.17.07.18;	author tgl;	state Exp;
branches;
next	1.46;

1.46
date	2005.03.07.02.00.28;	author tgl;	state Exp;
branches;
next	1.45;

1.45
date	2004.12.28.19.08.58;	author tgl;	state Exp;
branches;
next	1.44;

1.44
date	2004.12.03.17.46.19;	author tgl;	state Exp;
branches;
next	1.43;

1.43
date	2004.12.02.20.35.45;	author momjian;	state Exp;
branches;
next	1.42;

1.42
date	2004.11.15.06.32.14;	author neilc;	state Exp;
branches;
next	1.41;

1.41
date	2004.08.09.05.34.39;	author tgl;	state Exp;
branches;
next	1.40;

1.40
date	2004.05.21.05.07.55;	author tgl;	state Exp;
branches;
next	1.39;

1.39
date	2004.03.15.16.11.42;	author momjian;	state Exp;
branches;
next	1.38;

1.38
date	2004.03.15.16.06.43;	author momjian;	state Exp;
branches;
next	1.37;

1.37
date	2003.11.29.19.51.37;	author pgsql;	state Exp;
branches;
next	1.36;

1.36
date	2003.11.04.09.55.38;	author petere;	state Exp;
branches;
next	1.35;

1.35
date	2003.11.02.21.56.14;	author tgl;	state Exp;
branches
	1.35.2.1;
next	1.34;

1.34
date	2003.09.12.22.17.23;	author tgl;	state Exp;
branches;
next	1.33;

1.33
date	2003.08.31.17.32.19;	author petere;	state Exp;
branches;
next	1.32;

1.32
date	2003.04.06.17.47.24;	author tgl;	state Exp;
branches
	1.32.2.1;
next	1.31;

1.31
date	2003.03.13.01.30.29;	author petere;	state Exp;
branches;
next	1.30;

1.30
date	2002.11.08.20.26.12;	author tgl;	state Exp;
branches;
next	1.29;

1.29
date	2002.10.12.16.31.55;	author petere;	state Exp;
branches;
next	1.28;

1.28
date	2002.09.21.18.32.53;	author petere;	state Exp;
branches;
next	1.27;

1.27
date	2002.05.14.13.05.42;	author petere;	state Exp;
branches;
next	1.26;

1.26
date	2002.04.08.04.37.36;	author tgl;	state Exp;
branches;
next	1.25;

1.25
date	2002.01.20.22.19.56;	author petere;	state Exp;
branches;
next	1.24;

1.24
date	2002.01.07.02.29.13;	author petere;	state Exp;
branches;
next	1.23;

1.23
date	2001.12.04.01.49.17;	author tgl;	state Exp;
branches;
next	1.22;

1.22
date	2001.11.28.20.49.10;	author petere;	state Exp;
branches;
next	1.21;

1.21
date	2001.09.21.18.37.05;	author petere;	state Exp;
branches;
next	1.20;

1.20
date	2001.09.11.02.24.52;	author ishii;	state Exp;
branches;
next	1.19;

1.19
date	2001.09.09.23.52.12;	author petere;	state Exp;
branches;
next	1.18;

1.18
date	2001.08.06.22.53.26;	author tgl;	state Exp;
branches;
next	1.17;

1.17
date	2001.03.20.00.09.36;	author tgl;	state Exp;
branches;
next	1.16;

1.16
date	2001.01.02.05.56.02;	author tgl;	state Exp;
branches;
next	1.15;

1.15
date	2000.12.22.18.57.50;	author petere;	state Exp;
branches;
next	1.14;

1.14
date	2000.11.22.00.58.47;	author tgl;	state Exp;
branches;
next	1.13;

1.13
date	2000.11.21.23.40.27;	author petere;	state Exp;
branches;
next	1.12;

1.12
date	2000.10.22.19.11.05;	author tgl;	state Exp;
branches;
next	1.11;

1.11
date	2000.10.17.15.26.40;	author petere;	state Exp;
branches;
next	1.10;

1.10
date	2000.09.29.20.21.34;	author petere;	state Exp;
branches;
next	1.9;

1.9
date	2000.05.02.20.01.52;	author thomas;	state Exp;
branches;
next	1.8;

1.8
date	2000.04.02.22.28.54;	author tgl;	state Exp;
branches;
next	1.7;

1.7
date	2000.03.26.07.01.19;	author tgl;	state Exp;
branches;
next	1.6;

1.6
date	2000.01.09.20.54.35;	author tgl;	state Exp;
branches;
next	1.5;

1.5
date	99.12.07.22.41.40;	author momjian;	state Exp;
branches;
next	1.4;

1.4
date	99.05.20.02.46.40;	author tgl;	state Exp;
branches;
next	1.3;

1.3
date	98.12.29.02.24.19;	author thomas;	state Exp;
branches;
next	1.2;

1.2
date	98.07.29.06.50.02;	author thomas;	state Exp;
branches;
next	1.1;

1.1
date	98.03.01.08.16.10;	author thomas;	state Exp;
branches;
next	;

1.32.2.1
date	2003.09.07.04.36.43;	author momjian;	state Exp;
branches;
next	;

1.35.2.1
date	2003.11.04.09.45.29;	author petere;	state Exp;
branches;
next	;

1.67.2.1
date	2010.08.17.04.37.17;	author petere;	state Exp;
branches;
next	;


desc
@@


1.68
log
@Spell and markup checking
@
text
@<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.67 2010/05/02 12:22:40 sriggs Exp $ -->

 <chapter id="regress">
  <title>Regression Tests</title>

  <indexterm zone="regress">
   <primary>regression tests</primary>
  </indexterm>

  <indexterm zone="regress">
   <primary>test</primary>
  </indexterm>

  <para>
   The regression tests are a comprehensive set of tests for the SQL
   implementation in <productname>PostgreSQL</productname>.  They test
   standard SQL operations as well as the extended capabilities of
   <productname>PostgreSQL</productname>.
  </para>

  <sect1 id="regress-run">
   <title>Running the Tests</title>

  <para>
   The regression tests can be run against an already installed and
   running server, or using a temporary installation within the build
   tree.  Furthermore, there is a <quote>parallel</quote> and a
   <quote>sequential</quote> mode for running the tests.  The
   sequential method runs each test script alone, while the
   parallel method starts up multiple server processes to run groups
   of tests in parallel.  Parallel testing gives confidence that
   interprocess communication and locking are working correctly.
  </para>

  <para>
   To run the parallel regression tests after building but before installation,
   type:
<screen>
gmake check
</screen>
   in the top-level directory.  (Or you can change to
   <filename>src/test/regress</filename> and run the command there.)
   This will first build several auxiliary files, such as
   sample user-defined trigger functions, and then run the test driver
   script.  At the end you should see something like:
<screen>
<computeroutput>
=======================
 All 115 tests passed.
=======================
</computeroutput>
</screen>
   or otherwise a note about which tests failed.  See <xref
   linkend="regress-evaluation"> below before assuming that a
   <quote>failure</> represents a serious problem.
  </para>

   <para>
    Because this test method runs a temporary server, it will not work
    when you are the root user (since the server will not start as root).
    If you already did the build as root, you do not have to start all
    over.  Instead, make the regression test directory writable by
    some other user, log in as that user, and restart the tests.
    For example:
<screen>
<prompt>root# </prompt><userinput>chmod -R a+w src/test/regress</userinput>
<prompt>root# </prompt><userinput>su - joeuser</userinput>
<prompt>joeuser$ </prompt><userinput>cd <replaceable>top-level build directory</></userinput>
<prompt>joeuser$ </prompt><userinput>gmake check</userinput>
</screen>
    (The only possible <quote>security risk</quote> here is that other
    users might be able to alter the regression test results behind
    your back.  Use common sense when managing user permissions.)
   </para>
   <para>
    Alternatively, run the tests after installation.
   </para>

   <para>
    If you have configured <productname>PostgreSQL</productname> to install
    into a location where an older <productname>PostgreSQL</productname>
    installation already exists, and you perform <literal>gmake check</>
    before installing the new version, you might find that the tests fail
    because the new programs try to use the already-installed shared
    libraries.  (Typical symptoms are complaints about undefined symbols.)
    If you wish to run the tests before overwriting the old installation,
    you'll need to build with <literal>configure --disable-rpath</>.
    It is not recommended that you use this option for the final installation,
    however.
   </para>

   <para>
    The parallel regression test starts quite a few processes under your
    user ID.  Presently, the maximum concurrency is twenty parallel test
    scripts, which means forty processes: there's a server process and a
    <application>psql</> process for each test script.
    So if your system enforces a per-user limit on the number of processes,
    make sure this limit is at least fifty or so, else you might get
    random-seeming failures in the parallel test.  If you are not in
    a position to raise the limit, you can cut down the degree of parallelism
    by setting the <literal>MAX_CONNECTIONS</> parameter.  For example:
<screen>
gmake MAX_CONNECTIONS=10 check
</screen>
    runs no more than ten tests concurrently.
   </para>

  <para>
   To run the tests after installation<![%standalone-ignore;[ (see <xref linkend="installation">)]]>,
   initialize a data area and start the
   server, <![%standalone-ignore;[as explained in <xref linkend="runtime">, ]]> then type:
<screen>
gmake installcheck
</screen>
or for a parallel test:
<screen>
gmake installcheck-parallel
</screen>
   The tests will expect to contact the server at the local host and the
   default port number, unless directed otherwise by <envar>PGHOST</envar> and
   <envar>PGPORT</envar> environment variables.
  </para>

  <para>
   The source distribution also contains regression tests for the optional
   procedural languages and for some of the <filename>contrib</> modules.
   At present, these tests can be used only against an already-installed
   server.  To run the tests for all procedural languages that have been
   built and installed, change to the <filename>src/pl</> directory of the
   build tree and type:
<screen>
gmake installcheck
</screen>
   You can also do this in any of the subdirectories of <filename>src/pl</>
   to run tests for just one procedural language.  To run the tests for all
   <filename>contrib</> modules that have them, change to the
   <filename>contrib</> directory of the build tree and type:
<screen>
gmake installcheck
</screen>
   The <filename>contrib</> modules must have been built and installed first.
   You can also do this in a subdirectory of <filename>contrib</> to run
   the tests for just one module.
  </para>

  <para>
   The source distribution also contains regression tests of the static
   behaviour of Hot Standby. These tests require a running primary server
   and a running standby server that is accepting new WAL changes from the
   primary using either file-based log shipping or streaming replication.
   Those servers are not automatically created for you, nor is the setup
   documented here. Please check the various sections of the docs already
   devoted to the required commands and related issues.
  </para>

  <para>
   First create a database called "regression" on the primary.
<screen>
psql -h primary -c "CREATE DATABASE regression"
</screen>
   Next, run a preparatory script on the primary in the regression database:
   <filename>src/test/regress/sql/hs_primary_setup.sql</filename>, and
   allow for the changes to propagate to the standby, for example
<screen>
psql -h primary -f src/test/regress/sql/hs_primary_setup.sql regression
</screen>
   Now confirm that the default connection for the tester is the standby
   server under test and then run the <literal>standbycheck</> target from the regression
   directory:
<screen>
cd src/test/regress
gmake standbycheck
</screen>
  </para>

  <para>
   Some extreme behaviours can also be generated on the primary using the 
   script: <filename>src/test/regress/sql/hs_primary_extremes.sql</filename>
   to allow the behaviour of the standby to be tested.
  </para>

  <para>
   Additional automated testing may be available in later releases.
  </para>
  </sect1>

  <sect1 id="regress-evaluation">
   <title>Test Evaluation</title> 

   <para>
    Some properly installed and fully functional
    <productname>PostgreSQL</productname> installations can
    <quote>fail</quote> some of these regression tests due to
    platform-specific artifacts such as varying floating-point representation
    and message wording. The tests are currently evaluated using a simple
    <command>diff</command> comparison against the outputs
    generated on a reference system, so the results are sensitive to
    small system differences.  When a test is reported as
    <quote>failed</quote>, always examine the differences between
    expected and actual results; you might find that the
    differences are not significant.  Nonetheless, we still strive to
    maintain accurate reference files across all supported platforms,
    so it can be expected that all tests pass.
   </para>

   <para>
    The actual outputs of the regression tests are in files in the
    <filename>src/test/regress/results</filename> directory. The test
    script uses <command>diff</command> to compare each output
    file against the reference outputs stored in the
    <filename>src/test/regress/expected</filename> directory.  Any
    differences are saved for your inspection in
    <filename>src/test/regress/regression.diffs</filename>.  (Or you
    can run <command>diff</command> yourself, if you prefer.)
   </para>

   <para>
    If for some reason a particular platform generates a <quote>failure</>
    for a given test, but inspection of the output convinces you that
    the result is valid, you can add a new comparison file to silence
    the failure report in future test runs.  See
    <xref linkend="regress-variant"> for details.
   </para>

   <sect2>
    <title>Error message differences</title>
      
    <para>
     Some of the regression tests involve intentional invalid input
     values.  Error messages can come from either the
     <productname>PostgreSQL</productname> code or from the host
     platform system routines. In the latter case, the messages can
     vary between platforms, but should reflect similar
     information. These differences in messages will result in a
     <quote>failed</quote> regression test that can be validated by
     inspection.
    </para>
   </sect2>
    
   <sect2>
    <title>Locale differences</title>

    <para>
     If you run the tests against a server that was
     initialized with a collation-order locale other than C, then
     there might be differences due to sort order and subsequent
     failures.  The regression test suite is set up to handle this
     problem by providing alternate result files that together are
     known to handle a large number of locales.
    </para>

    <para>
     To run the tests in a different locale when using the
     temporary-installation method, pass the appropriate
     locale-related environment variables on
     the <command>make</command> command line, for example:
<programlisting>
gmake check LANG=de_DE.utf8
</programlisting>
     (The regression test driver unsets <envar>LC_ALL</envar>, so it
     does not work to choose the locale using that variable.)  To use
     no locale, either unset all locale-related environment variables
     (or set them to <literal>C</literal>) or use the following
     special invocation:
<programlisting>
gmake check NO_LOCALE=1
</programlisting>
     When running the tests against an existing installation, the
     locale setup is determined by the existing installation.  To
     change it, initialize the database cluster with a different
     locale by passing the appropriate options
     to <command>initdb</command>.
    </para>

    <para>
     In general, it is nevertheless advisable to try to run the
     regression tests in the locale setup that is wanted for
     production use, as this will exercise the locale- and
     encoding-related code portions that will actually be used in
     production.  Depending on the operating system environment, you
     might get failures, but then you will at least know what
     locale-specific behaviors to expect when running real
     applications.
    </para>
   </sect2>
    
   <sect2>
    <title>Date and time differences</title>

    <para>
     Most of the date and time results are dependent on the time zone
     environment.  The reference files are generated for time zone
     <literal>PST8PDT</literal> (Berkeley, California), and there will be
     apparent failures if the tests are not run with that time zone setting.
     The regression test driver sets environment variable
     <envar>PGTZ</envar> to <literal>PST8PDT</literal>, which normally
     ensures proper results.
    </para>
   </sect2>
    
   <sect2>
    <title>Floating-point differences</title>
      
    <para>
     Some of the tests involve computing 64-bit floating-point numbers (<type>double
     precision</type>) from table columns. Differences in
     results involving mathematical functions of <type>double
     precision</type> columns have been observed.  The <literal>float8</> and
     <literal>geometry</> tests are particularly prone to small differences
     across platforms, or even with different compiler optimization setting.
     Human eyeball comparison is needed to determine the real
     significance of these differences which are usually 10 places to
     the right of the decimal point.
    </para>

    <para>
     Some systems display minus zero as <literal>-0</>, while others
     just show <literal>0</>.
    </para>

    <para>
     Some systems signal errors from <function>pow()</function> and
     <function>exp()</function> differently from the mechanism
     expected by the current <productname>PostgreSQL</productname>
     code.
    </para>
   </sect2>

   <sect2>
    <title>Row ordering differences</title>
      
    <para>
You might see differences in which the same rows are output in a
different order than what appears in the expected file.  In most cases
this is not, strictly speaking, a bug.  Most of the regression test
scripts are not so pedantic as to use an <literal>ORDER BY</> for every single
<literal>SELECT</>, and so their result row orderings are not well-defined
according to the SQL specification.  In practice, since we are
looking at the same queries being executed on the same data by the same
software, we usually get the same result ordering on all platforms,
so the lack of <literal>ORDER BY</> is not a problem.  Some queries do exhibit
cross-platform ordering differences, however.  When testing against an
already-installed server, ordering differences can also be caused by
non-C locale settings or non-default parameter settings, such as custom values
of <varname>work_mem</> or the planner cost parameters.
    </para>

    <para>
Therefore, if you see an ordering difference, it's not something to
worry about, unless the query does have an <literal>ORDER BY</> that your
result is violating.  However, please report it anyway, so that we can add an
<literal>ORDER BY</> to that particular query to eliminate the bogus
<quote>failure</quote> in future releases.
    </para>

    <para>
You might wonder why we don't order all the regression test queries explicitly
to get rid of this issue once and for all.  The reason is that that would
make the regression tests less useful, not more, since they'd tend
to exercise query plan types that produce ordered results to the
exclusion of those that don't.
    </para>
   </sect2>

   <sect2>
    <title>Insufficient stack depth</title>
      
    <para>
     If the <literal>errors</literal> test results in a server crash
     at the <literal>select infinite_recurse()</> command, it means that
     the platform's limit on process stack size is smaller than the
     <![%standalone-ignore;[<xref linkend="guc-max-stack-depth">]]>
     <![%standalone-include;[<literal>max_stack_depth</literal>]]>
     parameter indicates.  This
     can be fixed by running the server under a higher stack
     size limit (4MB is recommended with the default value of 
     <varname>max_stack_depth</>).  If you are unable to do that, an
     alternative is to reduce the value of <varname>max_stack_depth</>.
    </para>
   </sect2>

   <sect2>
    <title>The <quote>random</quote> test</title>
      
    <para>
     The <literal>random</literal> test script is intended to produce 
     random results.   In rare cases, this causes the random regression
     test to fail.  Typing:
<programlisting>
diff results/random.out expected/random.out
</programlisting>
     should produce only one or a few lines of differences.  You need
     not worry unless the random test fails repeatedly.
    </para>
   </sect2>
  </sect1>

<!-- We might want to move the following section into the developer's guide. -->
  <sect1 id="regress-variant">
   <title>Variant Comparison Files</title>

   <para>
    Since some of the tests inherently produce environment-dependent
    results, we have provided ways to specify alternate <quote>expected</>
    result files.  Each regression test can have several comparison files
    showing possible results on different platforms.  There are two
    independent mechanisms for determining which comparison file is used
    for each test.
   </para>

   <para>
    The first mechanism allows comparison files to be selected for
    specific platforms.  There is a mapping file,
    <filename>src/test/regress/resultmap</filename>, that defines
    which comparison file to use for each platform.
    To eliminate bogus test <quote>failures</quote> for a particular platform,
    you first choose or make a variant result file, and then add a line to the
    <filename>resultmap</filename> file.
   </para>

   <para>
    Each line in the mapping file is of the form
<synopsis>
testname:output:platformpattern=comparisonfilename
</synopsis>
    The test name is just the name of the particular regression test
    module. The output value indicates which output file to check. For the 
    standard regression tests, this is always <literal>out</literal>. The
    value corresponds to the file extension of the output file.
    The platform pattern is a pattern in the style of the Unix
    tool <command>expr</> (that is, a regular expression with an implicit
    <literal>^</literal> anchor at the start).  It is matched against the
    platform name as printed by <command>config.guess</command>.
    The comparison file name is the base name of the substitute result
    comparison file.
   </para>

   <para>
    For example: some systems interpret very small floating-point values
    as zero, rather than reporting an underflow error.  This causes a
    few differences in the <filename>float8</> regression test.
    Therefore, we provide a variant comparison file,
    <filename>float8-small-is-zero.out</filename>, which includes
    the results to be expected on these systems.  To silence the bogus
    <quote>failure</quote> message on <systemitem>OpenBSD</systemitem>
    platforms, <filename>resultmap</filename> includes:
<programlisting>
float8:out:i.86-.*-openbsd=float8-small-is-zero.out
</programlisting>
    which will trigger on any machine where the output of
    <command>config.guess</command> matches <literal>i.86-.*-openbsd</literal>.
    Other lines
    in <filename>resultmap</> select the variant comparison file for other
    platforms where it's appropriate.
   </para>

   <para>
    The second selection mechanism for variant comparison files is
    much more automatic: it simply uses the <quote>best match</> among
    several supplied comparison files.  The regression test driver
    script considers both the standard comparison file for a test,
    <literal><replaceable>testname</>.out</>, and variant files named
    <literal><replaceable>testname</>_<replaceable>digit</>.out</>
    (where the <replaceable>digit</> is any single digit
    <literal>0</>-<literal>9</>).  If any such file is an exact match,
    the test is considered to pass; otherwise, the one that generates
    the shortest diff is used to create the failure report.  (If
    <filename>resultmap</filename> includes an entry for the particular
    test, then the base <replaceable>testname</> is the substitute
    name given in <filename>resultmap</filename>.)
   </para>

   <para>
    For example, for the <literal>char</literal> test, the comparison file
    <filename>char.out</filename> contains results that are expected
    in the <literal>C</> and <literal>POSIX</> locales, while
    the file <filename>char_1.out</filename> contains results sorted as
    they appear in many other locales.
   </para>

   <para>
    The best-match mechanism was devised to cope with locale-dependent
    results, but it can be used in any situation where the test results
    cannot be predicted easily from the platform name alone.  A limitation of
    this mechanism is that the test driver cannot tell which variant is
    actually <quote>correct</> for the current environment; it will just pick
    the variant that seems to work best.  Therefore it is safest to use this
    mechanism only for variant results that you are willing to consider
    equally valid in all contexts.
   </para>
    
  </sect1>

  <sect1 id="regress-coverage">
   <title>Test Coverage Examination</title>

   <para>
    The PostgreSQL source code can be compiled with coverage testing
    instrumentation, so that it becomes possible to examine which
    parts of the code are covered by the regression tests or any other
    test suite that is run with the code.  This is currently supported
    when compiling with GCC and requires the <command>gcov</command>
    and <command>lcov</command> programs.
   </para>

   <para>
    A typical workflow would look like this:
<screen>
./configure --enable-coverage ... OTHER OPTIONS ...
gmake
gmake check # or other test suite
gmake coverage-html
</screen>
    Then point your HTML browser
    to <filename>coverage/index.html</filename>.
    The <command>gmake</command> commands also work in subdirectories.
   </para>

   <para>
    To reset the execution counts between test runs, run:
<screen>
gmake coverage-clean
</screen>
   </para>
  </sect1>

</chapter>
@


1.67
log
@Docs for standbycheck regression tests.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.66 2010/04/03 07:22:55 petere Exp $ -->
d168 2
a169 2
   server under test and then run standbycheck from the regression
   directory.
@


1.67.2.1
log
@Spell and markup checking
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.67 2010/05/02 12:22:40 sriggs Exp $ -->
d168 2
a169 2
   server under test and then run the <literal>standbycheck</> target from the regression
   directory:
@


1.66
log
@Remove unnecessary xref endterm attributes and title ids

The endterm attribute is mainly useful when the toolchain does not support
automatic link target text generation for a particular situation.  In  the
past, this was required by the man page tools for all reference page links,
but that is no longer the case, and it now actually gets in the way of
proper automatic link text generation.  The only remaining use cases are
currently xrefs to refsects.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.65 2010/02/03 17:25:06 momjian Exp $ -->
d145 40
@


1.65
log
@Proofreading improvements for the Administration documentation book.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.64 2009/08/07 20:50:21 petere Exp $ -->
d4 1
a4 1
  <title id="regress-title">Regression Tests</title>
@


1.64
log
@Expand test coverage support to entire tree

Test coverage support now covers the entire source tree, including
contrib, instead of just src/backend.  In a related but independent
development, the commands make coverage and make coverage-html can be run
in any directory.

This turned out to be much easier than feared.  Besides a few ad hoc fixes
to pass the make target down the tree, change all affected makefiles to
list their directories in the SUBDIRS variable, changed from variants like
DIRS and WANTED_DIRS.  MSVC build fix was attempted as well.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.63 2009/04/27 16:27:36 momjian Exp $ -->
d29 1
a29 1
   sequential method runs each test script in turn, whereas the
d32 1
a32 4
   interprocess communication and locking are working correctly.  For
   historical reasons, the sequential test is usually run against an
   existing installation and the parallel method against a temporary
   installation, but there are no technical reasons for this.
d36 1
a36 1
   To run the regression tests after building but before installation,
d44 1
a44 1
   some sample user-defined trigger functions, and then run the test driver
d206 1
a206 1
     there might be differences due to sort order and follow-up
d208 1
a208 1
     problem by providing alternative result files that together are
d270 1
a270 1
     across platforms, or even with different compiler optimization options.
d298 1
a298 1
according to the letter of the SQL specification.  In practice, since we are
d300 2
a301 2
software, we usually get the same result ordering on all platforms, and
so the lack of <literal>ORDER BY</> isn't a problem.  Some queries do exhibit
d311 2
a312 2
result is violating.  But please report it anyway, so that we can add an
<literal>ORDER BY</> to that particular query and thereby eliminate the bogus
d364 1
a364 1
    results, we have provided ways to specify alternative <quote>expected</>
d410 1
a410 1
    which will trigger on any machine for which the output of
@


1.63
log
@Proofreading adjustments for first two parts of documentation (Tutorial
and SQL).
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.62 2009/02/12 13:26:03 petere Exp $ -->
d479 1
@


1.62
log
@Unset language-related locale settings in any case, otherwise psql will
speak in tongues and mess up the regression test diffs.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.61 2009/02/11 14:03:41 petere Exp $ -->
d40 1
a40 1
   type
d48 1
a48 1
   script.  At the end you should see something like
d67 1
a67 1
    For example
d104 1
a104 1
    by setting the <literal>MAX_CONNECTIONS</> parameter.  For example,
d114 1
a114 1
   server, <![%standalone-ignore;[as explained in <xref linkend="runtime">, ]]> then type
d118 1
a118 1
or for a parallel test
d133 1
a133 1
   build tree and type
d140 1
a140 1
   <filename>contrib</> directory of the build tree and type
d482 1
a482 1
    To reset the execution counts between test runs, run
@


1.61
log
@Only unset the locale environment when --no-locale is used (the way it was
presumably designed, but didn't act).  This allows running the temp install
tests in a non-C locale, thus exercising users' real environments better.
Document how to change locales for test runs.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.60 2008/09/05 12:11:18 petere Exp $ -->
d221 1
a221 1
gmake check LC_ALL=de_DE.utf8
d223 5
a227 1
     or analogously to use no locale:
d229 1
a229 1
gmake check LC_ALL=C
@


1.60
log
@Code coverage testing with gcov.  Documentation is in the regression test
chapter.

Author: Michelle Caisse <Michelle.Caisse@@Sun.COM>
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.59 2008/05/30 00:04:32 tgl Exp $ -->
d207 1
a207 1
     If you run the tests against an already-installed server that was
d214 30
@


1.59
log
@Copy refint.so and autoinc.so into the src/test/regress directory during
"make all", and then reference them there during the actual tests.  This
makes the handling of these files more parallel to that of regress.so,
and in particular simplifies use of the regression tests outside the
original build tree.  The PGDG and Red Hat RPMs have been doing this via
patches for a very long time.  Inclusion of the change in core was requested
by Jørgen Austvik of Sun, and I can't see any reason not to.

I attempted to fix the MSVC scripts for this too, but they may need
further tweaking ...
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.58 2007/06/12 17:49:12 mha Exp $ -->
d422 33
a454 1
  
@


1.58
log
@Update documentation of resultmap file format.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.57 2007/02/01 00:28:17 momjian Exp $ -->
d51 3
a53 3
======================
 All 100 tests passed.
======================
a69 1
<prompt>root# </prompt><userinput>chmod -R a+w contrib/spi</userinput>
@


1.57
log
@Consistenly use colons before '<programlisting>' blocks, where
appropriate.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.56 2007/01/31 20:56:18 momjian Exp $ -->
d354 1
a354 1
testname/platformpattern=comparisonfilename
d357 4
a360 1
    module.  The platform pattern is a pattern in the style of the Unix
d378 1
a378 1
float8/i.86-.*-openbsd=float8-small-is-zero
@


1.56
log
@Update documentation on may/can/might:

Standard English uses "may", "can", and "might" in different ways:

        may - permission, "You may borrow my rake."

        can - ability, "I can lift that log."

        might - possibility, "It might rain today."

Unfortunately, in conversational English, their use is often mixed, as
in, "You may use this variable to do X", when in fact, "can" is a better
choice.  Similarly, "It may crash" is better stated, "It might crash".

Also update two error messages mentioned in the documenation to match.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.55 2006/09/16 00:30:15 momjian Exp $ -->
d318 1
a318 1
     test to fail.  Typing
d373 1
a373 1
    platforms, <filename>resultmap</filename> includes
@


1.55
log
@Remove emacs info from footer of SGML files.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.54 2006/07/19 17:02:59 tgl Exp $ -->
d87 1
a87 1
    before installing the new version, you may find that the tests fail
d102 1
a102 1
    make sure this limit is at least fifty or so, else you may get
d164 1
a164 1
    expected and actual results; you may well find that the
d196 1
a196 1
     platform system routines. In the latter case, the messages may
d210 1
a210 1
     there may be differences due to sort order and follow-up
@


1.54
log
@Adjust spawn_process() to avoid unnecessary overhead processes: we can
just exec instead of creating a subprocess.  This reduces process usage
from four processes per parallel test to two.  I have no idea whether
a comparable optimization is possible or useful in the Windows port.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.53 2006/07/19 02:37:00 tgl Exp $ -->
a421 17

<!-- Keep this comment at the end of the file
Local variables:
mode:sgml
sgml-omittag:nil
sgml-shorttag:t
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:t
sgml-parent-document:nil
sgml-default-dtd-file:"./reference.ced"
sgml-exposed-tags:nil
sgml-local-catalogs:("/usr/lib/sgml/catalog")
sgml-local-ecat-files:nil
End:
-->
@


1.53
log
@Rewrite pg_regress as a C program instead of a shell script.
This allows it to be used on Windows without installing mingw
(though you do still need 'diff'), and opens the door to future
improvements such as message localization.
Magnus Hagander and Tom Lane.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.52 2006/06/18 15:38:36 petere Exp $ -->
d99 2
a100 3
    scripts, which means sixty processes: there's a server process, a
    <application>psql</>, and usually a shell parent process for the
    <application>psql</> for each test script.
d102 1
a102 1
    make sure this limit is at least seventy-five or so, else you may get
@


1.52
log
@Merge postmaster and postgres command into just postgres.  postmaster
symlink is kept for now for compatibility.  To call single-user mode, use
postgres --single.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.51 2006/04/06 18:54:36 petere Exp $ -->
a112 13
   <para>
    On some systems, the default Bourne-compatible shell
    (<filename>/bin/sh</filename>) gets confused when it has to manage
    too many child processes in parallel.  This may cause the parallel
    test run to lock up or fail.  In such cases, specify a different
    Bourne-compatible shell on the command line, for example:
<screen>
gmake SHELL=/bin/ksh check
</screen>
    If no non-broken shell is available, you may be able to work around the
    problem by limiting the number of connections, as shown above.
   </para>

d360 4
a363 7
    <literal>^</literal> anchor
    at the start).  It is matched against the platform name as printed
    by <command>config.guess</command> followed by
    <literal>:gcc</literal> or <literal>:cc</literal>, depending on
    whether you use the GNU compiler or the system's native compiler
    (on systems where there is a difference).  The comparison file
    name is the base name of the substitute result comparison file.
@


1.51
log
@Build src/test/regress/README during tarball making like the other
generated text files.  Fix build of that file, too.

Put the text files in the right place during make dist, so there are no
extra manual steps required anymore.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.50 2006/01/22 05:20:32 neilc Exp $ -->
d319 1
a319 1
     can be fixed by running the postmaster under a higher stack
@


1.50
log
@Allow an optional alias for the target table to be specified for UPDATE
and DELETE. If specified, the alias must be used instead of the full
table name. Also, the alias currently cannot be used in the SET clause
of UPDATE.

Patch from Atsushi Ogawa, various editorialization by Neil Conway.
Along the way, make the rowtypes regression test pass if add_missing_from
is enabled, and add a new (skeletal) regression test for DELETE.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.49 2005/10/18 21:43:33 tgl Exp $ -->
d316 3
a318 1
     <xref linkend="guc-max-stack-depth"> parameter indicates.  This
@


1.49
log
@Add an entry to the discussion of regression test failures about the
possibility of a failure due to stack overflow when max_stack_depth is
not set properly for the platform.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.48 2005/10/13 23:41:07 tgl Exp $ -->
d52 1
a52 1
 All 98 tests passed.
@


1.48
log
@Add documentation mentioning that there are separate regression tests
for the PL languages and for contrib.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.47 2005/07/24 17:07:18 tgl Exp $ -->
d310 15
@


1.47
log
@With the interval/day patch, the horology regression test no longer
fails near DST transition days, so remove the advice about that testing
problem.  Also improve the description of variant-comparison-file
selection.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.46 2005/03/07 02:00:28 tgl Exp $ -->
d141 22
@


1.46
log
@Add a note that regression test row ordering differences can be caused
by non-default planner parameter settings.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.45 2004/12/28 19:08:58 tgl Exp $ -->
d52 1
a52 1
 All 96 tests passed.
d151 1
a151 1
    and time zone support. The tests are currently evaluated using a simple
d173 8
d205 1
a205 21
     known to handle a large number of locales.  For example, for the
     <literal>char</literal> test, the expected file
     <filename>char.out</filename> handles the <literal>C</> and <literal>POSIX</> locales,
     and the file <filename>char_1.out</filename> handles many other
     locales.  The regression test driver will automatically pick the
     best file to match against when checking for success and for
     computing failure differences.  (This means that the regression
     tests cannot detect whether the results are appropriate for the
     configured locale.  The tests will simply pick the one result
     file that works best.)
    </para>

    <para>
     If for some reason the existing expected files do not cover some
     locale, you can add a new file.  The naming scheme is
     <literal><replaceable>testname</>_<replaceable>digit</>.out</>.
     The actual digit is not significant.  Remember that the
     regression test driver will consider all such files to be equally
     valid test results.  If the test results are platform-specific,
     the technique described in <xref linkend="regress-platform">
     should be used instead.
a212 21
     A few of the queries in the <filename>horology</filename> test will
     fail if you run the test on the day of a daylight-saving time
     changeover, or the day after one.  These queries expect that
     the intervals between midnight yesterday, midnight today and
     midnight tomorrow are exactly twenty-four hours &mdash; which is wrong
     if daylight-saving time went into or out of effect meanwhile.
    </para>

    <note>
     <para>
      Because USA daylight-saving time rules are used, this problem always
      occurs on the first Sunday of April, the last Sunday of October,
      and their following Mondays, regardless of when daylight-saving time
      is in effect where you live.  Also note that the problem appears or
      disappears at midnight Pacific time (UTC-7 or UTC-8), not midnight
      your local time.  Thus the failure may appear late on Saturday or
      persist through much of Tuesday, depending on where you live.
     </para>
    </note>

    <para>
d304 11
a314 2
  <sect1 id="regress-platform">
   <title>Platform-specific comparison files</title>
d317 7
a323 9
    Since some of the tests inherently produce platform-specific
    results, we have provided a way to supply platform-specific result
    comparison files.  Frequently, the same variation applies to
    multiple platforms; rather than supplying a separate comparison
    file for every platform, there is a mapping file that defines
    which comparison file to use.  So, to eliminate bogus test
    <quote>failures</quote> for a particular platform, you must choose
    or make a variant result file, and then add a line to the mapping
    file, which is <filename>src/test/regress/resultmap</filename>.
d340 1
a340 1
    name is the name of the substitute result comparison file.
d361 35
@


1.45
log
@More minor updates and copy-editing.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.44 2004/12/03 17:46:19 tgl Exp $ -->
d297 4
a300 2
cross-platform ordering differences, however.  (Ordering differences
can also be triggered by non-C locale settings.)
d305 2
a306 2
worry about, unless the query does have an <literal>ORDER BY</> that your result
is violating.  But please report it anyway, so that we can add an
d312 2
a313 2
You might wonder why we don't order all the regression test queries explicitly to
get rid of this issue once and for all.  The reason is that that would
@


1.44
log
@Improve description of --disable-rpath workaround for regression test
problems.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.43 2004/12/02 20:35:45 momjian Exp $ -->
d18 1
a18 3
   <productname>PostgreSQL</productname>.  From
   <productname>PostgreSQL</productname> 6.1 onward, the regression
   tests are current for every official release.
d25 1
a25 1
   The regression test can be run against an already installed and
d52 1
a52 1
 All 93 tests passed.
d57 2
a58 1
   linkend="regress-evaluation"> below for more.
d133 4
d138 2
a139 2
   default port number, unless directed otherwise by <envar>PGHOST</envar> and <envar>PGPORT</envar>
   environment variables.
d329 1
a329 1
     not worry unless the random test repeatedly fails.
@


1.43
log
@Add mention of using --disable-rpath with regression test.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.42 2004/11/15 06:32:14 neilc Exp $ -->
a49 4
   script. On some platforms if you want perform a <literal>make check<literal>
   without installing it first, you have to build with:
   <literal>configure --disable-rpath</>.
   At the end you should see something like
d85 13
@


1.42
log
@Replace "--" and "---" with "&mdash;" as appropriate, for better-looking
output.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.41 2004/08/09 05:34:39 tgl Exp $ -->
d49 5
a53 2
   some sample
   user-defined trigger functions, and then run the test driver
@


1.41
log
@Remove no-longer-relevant information about trying to force your OS
to support PST8PDT time zone for the regression tests.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/regress.sgml,v 1.40 2004/05/21 05:07:55 tgl Exp $ -->
d214 1
a214 1
     midnight tomorrow are exactly twenty-four hours --- which is wrong
@


1.40
log
@Integrate src/timezone library for all platforms.  There is more we can
and should do now that we control our own destiny for timezone handling,
but this commit gets the bulk of the picayune diffs in place.
Magnus Hagander and Tom Lane.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/regress.sgml,v 1.39 2004/03/15 16:11:42 momjian Exp $ -->
d233 2
a234 2
     <literal>PST8PDT</literal> (Berkeley, California), and there will be apparent
     failures if the tests are not run with that time zone setting.
d237 1
a237 29
     ensures proper results.  However, your operating system must provide
     support for the <literal>PST8PDT</literal> time zone, or the time zone-dependent
     tests will fail. To verify that your machine does have this
     support, type the following:
<screen>
env TZ=PST8PDT date
</screen>
     The command above should have returned the current system time in
     the <literal>PST8PDT</literal> time zone. If the <literal>PST8PDT</literal> time zone is not available,
     then your system may have returned the time in UTC. If the
     <literal>PST8PDT</literal> time zone is missing, you can set the time zone
     rules explicitly:
<programlisting>
PGTZ='PST8PDT7,M04.01.0,M10.05.03'; export PGTZ
</programlisting>
    </para>

    <para>
     There appear to be some systems that do not accept the
     recommended syntax for explicitly setting the local time zone
     rules; you may need to use a different <envar>PGTZ</envar>
     setting on such machines.
    </para>

    <para>
     Some systems using older time-zone libraries fail to apply
     daylight-saving corrections to dates before 1970, causing
     pre-1970 <acronym>PDT</acronym> times to be displayed in <acronym>PST</acronym> instead.  This will
     result in localized differences in the test results.
@


1.39
log
@Update random regression test text to be clearer.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/regress.sgml,v 1.38 2004/03/15 16:06:43 momjian Exp $ -->
d381 3
a383 4
    For example: some systems using older time zone libraries fail to apply
    daylight-saving corrections to dates before 1970, causing
    pre-1970 <acronym>PDT</acronym> times to be displayed in <acronym>PST</acronym> instead.  This causes a
    few differences in the <filename>horology</> regression test.
d385 1
a385 1
    <filename>horology-no-DST-before-1970.out</filename>, which includes
d387 2
a388 2
    <quote>failure</quote> message on <systemitem>HPUX</systemitem> platforms,
    <filename>resultmap</filename> includes
d390 1
a390 1
horology/.*-hpux=horology-no-DST-before-1970
d393 1
a393 1
    <command>config.guess</command> includes <literal>-hpux</literal>.
@


1.38
log
@Update random test description because it fails less frequently now.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql-server/doc/src/sgml/regress.sgml,v 1.37 2003/11/29 19:51:37 pgsql Exp $ -->
d335 3
a337 3
     There is at least one case in the <literal>random</literal> test
     script that is intended to produce random results. This causes
     random to fail the regression test occasionally.  Typing
d342 1
a342 2
     not worry unless the random test always fails in repeated
     attempts.
@


1.37
log
@
$Header: -> $PostgreSQL Changes ...
@
text
@d1 1
a1 1
<!-- $PostgreSQL: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.36 2003/11/04 09:55:38 petere Exp $ -->
d337 1
a337 2
     random to fail the regression test once in a while (perhaps once
     in every five to ten trials).  Typing
d343 1
a343 4
     attempts.  (On the other hand, if the random test is
     <emphasis>never</emphasis> reported to fail even in many trials
     of the regression tests, you probably <emphasis>should</emphasis>
     worry.)
@


1.36
log
@Random copy-editing.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.35 2003/11/02 21:56:14 tgl Exp $ -->
@


1.35
log
@Provide a way to run the parallel regression tests with a user-specified
limit on the number of simultaneous connections.  Andrew Dunstan, with
review by Tom Lane.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.34 2003/09/12 22:17:23 tgl Exp $ -->
a62 1
  <note>
a83 1
  </note>
a84 1
  <tip>
a100 1
  </tip>
a101 1
  <tip>
a113 1
  </tip>
d220 1
a220 1
      Because USA daylight-saving rules are used, this problem always
d222 1
a222 1
      and their following Mondays, regardless of when daylight-saving
@


1.35.2.1
log
@Random copy-editing.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.35 2003/11/02 21:56:14 tgl Exp $ -->
d63 1
d85 1
d87 1
d104 1
d106 1
d119 1
d226 1
a226 1
      Because USA daylight-saving time rules are used, this problem always
d228 1
a228 1
      and their following Mondays, regardless of when daylight-saving time
@


1.34
log
@Update obsolete examples of error messages; various other minor editing.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.33 2003/08/31 17:32:19 petere Exp $ -->
d55 1
a55 1
 All 77 tests passed.
d59 1
a59 1
   or otherwise a note about what tests failed.  See <xref
d66 1
a66 1
    when you are the root user (the server will not start as root).
d91 3
a93 2
    scripts, which means sixty processes: there's a server process, a <application>psql</>,
    and usually a shell parent process for the <application>psql</> for each test script.
d97 6
a102 3
    a position to raise the limit, you can edit the file
    <filename>src/test/regress/parallel_schedule</> to split the
    larger concurrent test sets into more manageable groups.
d116 2
a117 2
    If no non-broken shell is available, you can alter the parallel test
    schedule as suggested above.
d184 1
a184 1
     initialized with a collation order locale different than C then
d206 1
a206 1
     valid test results.  If the test results are platform-dependent,
d252 1
a252 1
     then your system may have returned the time in GMT. If the
d371 1
a371 1
    file, which is <filename>resultmap</filename>.
@


1.33
log
@Add/edit index entries.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.32 2003/04/06 17:47:24 tgl Exp $ -->
d395 2
a396 2
    <quote>failure</quote> message on <systemitem>HPPA</systemitem> platforms, <filename>resultmap</filename>
    includes
d398 1
a398 1
horology/hppa=horology-no-DST-before-1970
d400 3
a402 2
    which will trigger on any machine for which the output of <command>config.guess</command>
    begins with <literal>hppa</literal>.  Other lines
@


1.32
log
@Clarify exactly when DST-changeover-induced regression test failures
can be expected to occur.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.31 2003/03/13 01:30:29 petere Exp $ -->
d5 8
@


1.32.2.1
log
@Update this branch to match CVS head, includes WIN32 improvements.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.33 2003/08/31 17:32:19 petere Exp $ -->
a4 8

  <indexterm zone="regress">
   <primary>regression tests</primary>
  </indexterm>

  <indexterm zone="regress">
   <primary>test</primary>
  </indexterm>
@


1.31
log
@Big editing for consistent content and presentation.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.30 2002/11/08 20:26:12 tgl Exp $ -->
d204 1
a204 1
     Some of the queries in the <filename>horology</filename> test will
d206 2
a207 2
     changeover, or the day before or after one.  These queries assume
     that the intervals between midnight yesterday, midnight today and
d211 12
@


1.30
log
@Update discussion of floating-point problems in regression tests.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.29 2002/10/12 16:31:55 petere Exp $ -->
a5 3
  <sect1 id="regress-intro">
   <title>Introduction</title>

d10 3
a12 5
   <productname>PostgreSQL</productname>.  The test suite was
   originally developed by Jolly Chen and Andrew Yu, and was
   extensively revised and repackaged by Marc Fournier and Thomas
   Lockhart.  From <productname>PostgreSQL</productname> 6.1 onward
   the regression tests are current for every official release.
a14 2
  </sect1>

d36 1
a36 1
<prompt>$ </prompt><userinput>gmake check</userinput>
d41 1
a41 1
   platform-dependent <quote>expected</quote> files and some sample
d62 1
a62 1
    For example,
d83 1
a83 1
    scripts, which means sixty processes --- there's a backend, a <application>psql</>,
a100 1
    <informalexample>
d102 1
a102 1
<prompt>$ </prompt><userinput>gmake SHELL=/bin/ksh check</userinput>
a103 1
    </informalexample>
d114 1
a114 1
<prompt>$ </prompt><userinput>gmake installcheck</userinput>
d131 1
a131 1
    <application>diff</application> comparison against the outputs
d144 1
a144 1
    script uses <application>diff</application> to compare each output
d149 1
a149 1
    can run <application>diff</application> yourself, if you prefer.)
d177 1
a177 1
     <quote>char</quote> test, the expected file
d208 1
a208 1
     midnight tomorrow are exactly twenty-four hours -- which is wrong
d215 1
a215 1
     <literal>PST8PDT</literal> (Berkeley, California) and there will be apparent
d219 1
a219 1
     ensures proper results.  However, your system must provide library
d224 1
a224 1
<prompt>$ </prompt><userinput>env TZ=PST8PDT date</userinput>
d227 1
a227 1
     the <literal>PST8PDT</literal> time zone. If the <literal>PST8PDT</literal> database is not available,
d229 1
a229 1
     <literal>PST8PDT</literal> time zone is not available, you can set the time zone
d244 1
a244 1
     Some systems using older time zone libraries fail to apply
d255 2
a256 2
     Some of the tests involve computing 64-bit (<type>double
     precision</type>) numbers from table columns. Differences in
d286 2
a287 2
scripts are not so pedantic as to use an ORDER BY for every single
SELECT, and so their result row orderings are not well-defined
d291 1
a291 1
so the lack of ORDER BY isn't a problem.  Some queries do exhibit
d298 1
a298 1
worry about, unless the query does have an ORDER BY that your result
d300 1
a300 1
ORDER BY to that particular query and thereby eliminate the bogus
d305 1
a305 1
You might wonder why we don't order all the regress test queries explicitly to
d317 1
a317 1
     There is at least one case in the <quote>random</quote> test
d356 2
a357 2
    module.  The platform pattern is a pattern in the style of
    <citerefentry><refentrytitle>expr</><manvolnum>1</></citerefentry> (that is, a regular expression with an implicit
d360 1
a360 1
    by <filename>config.guess</filename> followed by
d381 1
a381 1
    begins with <quote><literal>hppa</literal></quote>.  Other lines
@


1.29
log
@Fix linking problem.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.28 2002/09/21 18:32:53 petere Exp $ -->
d267 3
a269 3
     precision</type> columns have been observed.  The float8 and
     geometry tests are particularly prone to small differences across
     platforms, or even with different compiler optimization options.
d276 5
a284 26
    </para>
   </sect2>
    
   <sect2>
    <title>Polygon differences</title>
      
    <para>
     Several of the tests involve operations on geographic data about
     the Oakland/Berkeley, California street map. The map data is expressed as
     polygons whose vertices are represented as pairs of <type>double
     precision</type> numbers (decimal latitude and
     longitude). Initially, some tables are created and loaded with
     geographic data, then some views are created that join two
     tables using the polygon intersection operator
     (<literal>##</literal>), then a select is done on the view.
    </para>

    <para>
     When comparing the results from different platforms, differences
     occur in the 2nd or 3rd place to the right of the decimal
     point. The SQL statements where these problems occur are the
     following:
<programlisting>
SELECT * from street;
SELECT * from iexit;
</programlisting>
@


1.28
log
@Add more appropriate markup.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql-server/doc/src/sgml/regress.sgml,v 1.27 2002/05/14 13:05:42 petere Exp $ -->
a363 1
<![%standalone-ignore;[
a416 1
]]>
@


1.27
log
@Make regression tests locale-proof by setting some locale categories
to C at run-time, and providing alternative output files for different
sort orders.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.26 2002/04/08 04:37:36 tgl Exp $ -->
d90 2
a91 2
    scripts, which means sixty processes --- there's a backend, a psql,
    and usually a shell parent process for the psql for each test script.
d187 1
a187 1
     <filename>char.out</filename> handles the C and POSIX locales,
@


1.26
log
@DST-transition-sensitive tests seem to be in horology now, not
timestamp.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.25 2002/01/20 22:19:56 petere Exp $ -->
d180 15
a194 7
     The tests expect to run in plain <quote>C</quote> locale.  This
     should not cause any problems when you run the tests against a
     temporary installation, since the regression test driver takes care
     to start the server in C locale.  However, if you run the tests
     against an already-installed server that is using non-C locale settings,
     you may see differences caused by varying rules for string sort order,
     formatting of numeric and monetary values, and so forth.
d198 8
a205 6
     In some locales the resulting differences are small and easily checked by
     inspection.  However, in a locale that changes the rules for formatting
     of numeric values (typically by swapping the usage of commas and
     decimal points), entry of some data values will fail, resulting in
     extensive differences later in the tests where the missing data values
     are supposed to be used.
@


1.25
log
@Spell-check and markup police
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.24 2002/01/07 02:29:13 petere Exp $ -->
d203 1
a203 1
     Some of the queries in the <filename>timestamp</filename> test will
@


1.24
log
@Editorial review
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.23 2001/12/04 01:49:17 tgl Exp $ -->
d89 1
a89 1
    userid.  Presently, the maximum concurrency is twenty parallel test
@


1.23
log
@Document max-processes-per-user limit as a possible cause of failures
in the parallel regression tests.  Suggest editing parallel_schedule
as a workaround if one cannot fix the system limits.
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.22 2001/11/28 20:49:10 petere Exp $ -->
d138 1
a138 1
    platform-specific artifacts such as varying floating point representation
d251 1
a251 1
    <title>Floating point differences</title>
@


1.22
log
@proof-reading
@
text
@d1 1
a1 1
<!-- $Header: /cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.21 2001/09/21 18:37:05 petere Exp $ -->
d88 15
d113 2
@


1.21
log
@Readd test/regress/README file, this time with a well-defined and simple
rule to remake it when necessary.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.20 2001/09/11 02:24:52 ishii Exp $ -->
d74 1
a74 1
<prompt>joeuser$ </prompt><userinput>cd &lt;build top-level directory&gt;</userinput>
d187 1
a187 1
     fail if you run the test on the day of a daylight-savings time
d191 1
a191 1
     if daylight-savings time went into or out of effect meanwhile.
d227 1
a227 1
     daylight-savings corrections to dates before 1970, causing
d373 1
a373 1
    daylight-savings corrections to dates before 1970, causing
@


1.20
log
@Fix regression test in the case of building as root.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.19 2001/09/09 23:52:12 petere Exp $ -->
d6 2
a7 5
  <abstract>
   <para>
    Regression test instructions and analysis
   </para>
  </abstract>
d20 5
d102 3
a104 3
   To run the tests after installation (see <xref
   linkend="installation">), initialize a data area and start the
   server, as explained in <xref linkend="runtime">, then type
d112 1
d261 1
a261 1
     the Oakland/Berkeley, CA street map. The map data is expressed as
d283 1
a283 1
    <title>Tuple ordering differences</title>
d286 1
a286 1
You might see differences in which the same tuples are output in a
d290 2
a291 2
SELECT, and so their result tuple orderings are not well-defined
according to the letter of the SQL spec.  In practice, since we are
d331 1
a331 1
     of the regress tests, you probably <emphasis>should</emphasis>
d337 1
d391 1
@


1.19
log
@Markup additions and spell check. (covers Admin Guide)
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.18 2001/08/06 22:53:26 tgl Exp $ -->
d70 1
d72 1
@


1.18
log
@Update now-obsolete example of platform-specific regression comparison
files.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.17 2001/03/20 00:09:36 tgl Exp $ -->
d105 1
a105 1
   default port number, unless directed otherwise by PGHOST and PGPORT
d181 1
a181 1
     Some of the queries in the <quote>timestamp</quote> test will
d192 1
a192 1
     PST8PDT (Berkeley, California) and there will be apparent
d197 1
a197 1
     support for the PST8PDT time zone, or the time zone-dependent
d204 1
a204 1
     the PST8PDT time zone. If the PST8PDT database is not available,
d206 1
a206 1
     PST8PDT time zone is not available, you can set the time zone
d223 1
a223 1
     pre-1970 PDT times to be displayed in PST instead.  This will
d303 1
a303 1
You might wonder why we don't ORDER all the regress test SELECTs to
d355 1
a355 1
    expr(1) (that is, a regular expression with an implicit
d368 1
a368 1
    pre-1970 PDT times to be displayed in PST instead.  This causes a
d373 1
a373 1
    <quote>failure</quote> message on HPPA platforms, resultmap
d378 1
a378 1
    which will trigger on any machine for which config.guess's output
d380 1
a380 1
    in resultmap select the variant comparison file for other
@


1.17
log
@Discuss LOCALE differences as a reason for regression test failure.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.16 2001/01/02 05:56:02 tgl Exp $ -->
d52 1
a52 1
 All 76 tests passed.
d179 10
a188 1
      
d195 2
a196 2
     <envar>PGTZ</envar> to <literal>PST8PDT</literal> to ensure
     proper results.  However, your system must provide library
a225 9
      
    <para>
     Some of the queries in the <quote>timestamp</quote> test will
     fail if you run the test on the day of a daylight-savings time
     changeover, or the day before or after one.  These queries assume
     that the intervals between midnight yesterday, midnight today and
     midnight tomorrow are exactly twenty-four hours -- which is wrong
     if daylight-savings time went into or out of effect meanwhile.
    </para>
d366 4
a369 11
    For example: the int2 regression test includes a deliberate entry
    of a value that is too large to fit in int2.  The specific error
    message that is produced is platform-dependent; our reference
    platform emits
<screen>
<computeroutput>ERROR:  pg_atoi: error reading "100000": Numerical result out of range</computeroutput>
</screen>
    but a fair number of other Unix platforms emit
<screen>
<computeroutput>ERROR:  pg_atoi: error reading "100000": Result too large</computeroutput>
</screen>
d371 2
a372 2
    <filename>int2-too-large.out</filename>, that includes this
    spelling of the error message.  To silence the bogus
d376 1
a376 1
int2/hppa=int2-too-large
@


1.16
log
@Document tuple ordering differences as a possible cause of
regression test 'failures'.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.15 2000/12/22 18:57:50 petere Exp $ -->
d52 1
a52 1
 All 75 tests passed.
d67 1
d104 3
a106 2
   The server is expected to be running on the local host with the
   default port number.
d116 2
a117 2
    artifacts of floating point representation and time zone
    support. The tests are currently evaluated using a simple
d155 23
d290 2
a291 1
cross-platform ordering differences, however.
d296 2
a297 2
worry about (unless the query does have an ORDER BY that your result
is violating).  But please report it anyway, so that we can add an
@


1.15
log
@Replace incorrect uses of 'which' with 'that'.  (so-called "wicked which")
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.14 2000/11/22 00:58:47 tgl Exp $ -->
d251 34
a284 1
    
@


1.14
log
@Tweak resultmap documentation, add to README too.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.13 2000/11/21 23:40:27 petere Exp $ -->
d147 1
a147 1
     <quote>failed</quote> regression test which can be validated by
d180 1
a180 1
     There appear to be some systems which do not accept the
d235 1
a235 1
     geographic data, then some views are created which join two
@


1.13
log
@Allow for different result files when using GCC versus native compiler.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.12 2000/10/22 19:11:05 tgl Exp $ -->
d292 1
a292 1
testname/platformnamepattern=comparisonfilename
d295 3
a297 2
    module.  The platform name pattern is a pattern in the style of
    expr(1) (that is, a regular expression with an implicit ^ anchor
d299 1
a299 1
    by <filename>config.guess</filename> with an appended
@


1.12
log
@Fix typo.
@
text
@d1 1
a1 1
<!-- $Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/regress.sgml,v 1.11 2000/10/17 15:26:40 petere Exp $ -->
d275 1
a275 1
    <title>Platform-specific comparison files</title>
d277 53
a329 44
     <para>
  Since some of the tests inherently produce platform-specific results,
  we have provided a way to supply platform-specific result comparison
  files.  Frequently, the same variation applies to multiple platforms;
  rather than supplying a separate comparison file for every platform,
  there is a mapping file that defines which comparison file to use.
  So, to eliminate bogus test "failures" for a particular platform,
  you must choose or make a variant result file, and then add a line
  to the mapping file, which is "resultmap".
     </para>

     <para>
  Each line in the mapping file is of the form
	<programlisting>
		testname/platformnamepattern=comparisonfilename
	</programlisting>
  The test name is just the name of the particular regression test module.
  The platform name pattern is a pattern in the style of expr(1) (that is,
  a regular expression with an implicit ^ anchor at the start).  It is matched
  against the platform name as printed by config.guess.  The comparison
  file name is the name of the substitute result comparison file.
     </para>

     <para>
  For example: the int2 regress test includes a deliberate entry of a value
  that is too large to fit in int2.  The specific error message that is
  produced is platform-dependent; our reference platform emits
	<programlisting>
    ERROR:  pg_atoi: error reading "100000": Numerical result out of range
	</programlisting>
  but a fair number of other Unix platforms emit
	<programlisting>
    ERROR:  pg_atoi: error reading "100000": Result too large
	</programlisting>
  Therefore, we provide a variant comparison file, int2-too-large.out,
  that includes this spelling of the error message.  To silence the
  bogus "failure" message on HPPA platforms,  resultmap includes
	<programlisting>
		int2/hppa=int2-too-large
	</programlisting>
  which will trigger on any machine for which config.guess's output
  begins with 'hppa'.  Other lines in resultmap select the variant
  comparison file for other platforms where it's appropriate.
     </para>
@


1.11
log
@* doc/src/sgml/regress.sgml: Update for new driver script.

* doc/src/sgml/installation.sgml: ditto.

* src/test/regress/README: Regenerate.

* doc/src/sgml/docguide.sgml: Explain how it was done.  Explain how
INSTALL and HISTORY are (now) generated.

* doc/src/sgml/Makefile: Implement HISTORY generation to be analoguous
to INSTALL.
@
text
@d1 1
a1 1
<!-- $Header$ -->
d33 2
a34 3
   existing installation and the parallel method
   <quote>stand-alone</quote>, but there are technical reasons for
   this.
@


1.10
log
@Add `id' attributes to all `chapter' and `sect1' tags, to generate useful
names for the HTML files (e.g., not x4856.htm).
@
text
@d1 2
d4 1
a4 1
  <title id="regress-title">Regression Test</title>
d8 1
a8 1
    Regression test instructions and analysis.
d13 8
a20 16
   The PostgreSQL regression tests are a comprehensive set of tests for the
   SQL implementation embedded in PostgreSQL.  They test standard SQL
   operations as well as the extended capabilities of PostgreSQL.
  </para>

  <para>
   There are two different ways in which the regression tests can be run:
   the "sequential" method and the "parallel" method.  The sequential method
   runs each test script in turn, whereas the parallel method starts up
   multiple server processes to run groups of tests in parallel.  Parallel
   testing gives confidence that interprocess communication and locking
   are working correctly.  Another key difference is that the sequential
   test procedure uses an already-installed postmaster, whereas the
   parallel test procedure tests a system that has been built but not yet
   installed.  (The parallel test script actually does an installation into
   a temporary directory and fires up a private postmaster therein.)
d24 12
a35 9
   Some properly installed and fully functional PostgreSQL installations
   can "fail" some of these regression tests due to artifacts of floating point
   representation and time zone support. The tests are currently evaluated
   using a simple <application>diff</application> comparison against the
   outputs generated on a reference system, so the results are sensitive to
   small system differences.
   When a test is reported as "failed", always examine the differences
   between expected and actual results; you may well find that the differences
   are not significant.
d39 20
a58 4
   The regression tests were originally developed by Jolly Chen and Andrew Yu,
   and were extensively revised/repackaged by Marc Fournier and Thomas Lockhart.
   From <productname>PostgreSQL</productname> v6.1 onward
   the regression tests are current for every official release. 
d61 1
a61 3
  <sect1 id="regress-environment">
   <title>Regression Environment</title>

d63 13
a75 28
    The regression testing notes below assume the following (except where noted):
    <itemizedlist spacing="compact" mark="bullet">
     <listitem>
      <para>
       Commands are Unix-compatible. See note below.
      </para>
     </listitem>
     <listitem>
      <para>
       Defaults are used except where noted.
      </para>
     </listitem>
     <listitem>
      <para>
       User postgres is the <productname>Postgres</productname> superuser.
      </para>
     </listitem>
     <listitem>
      <para>
       The source path is /usr/src/pgsql (other paths are possible).
      </para>
     </listitem>
     <listitem>
      <para>
       The runtime path is /usr/local/pgsql (other paths are possible).
      </para>
     </listitem>
    </itemizedlist>
a76 1

d78 1
a78 4
    Normally, the regression tests should be run as the postgres user since
    the 'src/test/regress' directory and sub-directories are owned by the
    postgres user. If you run the regression test as another user the
    'src/test/regress' directory tree must be writeable by that user.
d80 1
d82 1
d84 10
a93 14
    It was formerly necessary to run the postmaster with system time zone
    set to PST, but this is no longer required.  You can run the regression
    tests under your normal postmaster configuration.  The test script will
    set the PGTZ environment variable to ensure that timezone-dependent tests
    produce the expected results.  However, your system must provide
    library support for the PST8PDT time zone, or the timezone-dependent
    tests will fail.
    To verify that your machine does have this support, type
    the following:

    <programlisting>
setenv TZ PST8PDT
date
    </programlisting>
d95 15
d112 13
a124 7
    The "date" command above should have returned the current system time
    in the PST8PDT time zone. If the PST8PDT database is not available, then
    your system may have returned the time in GMT. If the PST8PDT time zone
    is not available, you can set the time zone rules explicitly:
    <programlisting>
setenv PGTZ PST8PDT7,M04.01.0,M10.05.03
    </programlisting>
d128 9
a136 1
    The directory layout for the regression test area is:
d138 3
a140 78
    <table tocentry="1">
     <title>Directory Layout</title>

     <titleabbrev>Kerberos</titleabbrev>

     <tgroup cols="2">
      <thead>
       <row>
	<entry>Directory</entry>
	<entry>Description</entry>
       </row>
      </thead>
      <tbody>
       <row>
	<entry>Directory</entry>
	<entry>Description</entry>
       </row>
       <row>
	<entry>input</entry>
	<entry>
	 Source files that are converted using
	 <command>make all</command> into
	 some of the <filename>.sql</filename> files in the
	 <filename>sql</filename> subdirectory.
	</entry>
       </row>

       <row>
	<entry>output</entry>
	<entry>
	 Source files that are converted using
	 <command>make all</command> into
	 <filename>.out</filename> files in the
	 <filename>expected</filename> subdirectory.
	</entry>
       </row>

       <row>
	<entry>sql</entry>
	<entry>
	 <filename>.sql</filename> files used to perform the
	 regression tests.
	</entry>
       </row>

       <row>
	<entry>expected</entry>
	<entry>
	 <filename>.out</filename> files that represent what we
	 <emphasis>expect</emphasis> the results to
	 look like.
	</entry>
       </row>

       <row>
	<entry>results</entry>
	<entry>
	 <filename>.out</filename> files that contain what the results
	 <emphasis>actually</emphasis> look
	 like. Also used as temporary storage for table copy testing.
	</entry>
       </row>

       <row>
	<entry>tmp_check</entry>
	<entry>
	 Temporary installation created by parallel testing script.
	</entry>
       </row>
      </tbody>
     </tgroup>
    </table>
   </para>
  </sect1>
  
  <sect1 id="regress-procedure">
    <title>Regression Test Procedure</title>
    
d142 8
a149 5
      Commands were tested on RedHat Linux version 4.2 using the bash shell.
      Except where noted, they will probably work on most systems. Commands
      like <filename>ps</filename> and <filename>tar</filename> vary
    wildly on what options you should use on each
      platform. <emphasis>Use common sense</emphasis> before typing in these commands.
d151 1
d153 2
a154 2
    <procedure>
      <title><productname>Postgres</productname> Regression Test</title>
d156 29
a184 121
      <step performance="required">
	<para>
	  Prepare the files needed for the regression test with:
	  <programlisting>
	    cd /usr/src/pgsql/src/test/regress
	    gmake clean
	    gmake all
	  </programlisting>
	  You can skip "gmake clean" if this is the first time you
	  are running the tests.
	</para>
	<para>
	  This step compiles a <acronym>C</acronym>
	  program with PostgreSQL extension functions into a shared library.
	  Localized SQL scripts and output-comparison files are also created
	  for the tests that need them.  The localization replaces macros in
	  the source files with absolute pathnames and user names.
	</para>
      </step>

      <step performance="optional">
	<para>
	  If you intend to use the "sequential" test procedure, which tests
	  an already-installed postmaster, be sure that the postmaster
	  is running.  If it isn't already running,
	  start the postmaster in an available window by typing
	  <programlisting>
	    postmaster
	  </programlisting>
	  or start the postmaster daemon running in the background by typing
	  <programlisting>
	    cd
	    nohup postmaster > regress.log 2>&1 &
	  </programlisting>
	  The latter is probably preferable, since the regression test log
	  will be quite lengthy (60K or so, in
	  <productname>Postgres</productname> 7.0) and you might want to
	  review it for clues if things go wrong.
	  
	  <note>
	    <para>
	      Do not run <filename>postmaster</filename> from the root account.
	    </para>
	  </note>
	</para>
      </step>
      
      <step performance="required">
	<para>
	  Run the regression tests.  For a sequential test, type
	  <programlisting>
	    cd /usr/src/pgsql/src/test/regress
	    gmake runtest
	  </programlisting>
	  For a parallel test, type
	  <programlisting>
	    cd /usr/src/pgsql/src/test/regress
	    gmake runcheck
	  </programlisting>
	  The sequential test just runs the test scripts using your
	  already-running postmaster.
	  The parallel test will perform a complete installation of
	  <productname>Postgres</productname> into a temporary directory,
	  start a private postmaster therein, and then run the test scripts.
	  Finally it will kill the private postmaster (but the temporary
	  directory isn't removed automatically).
	</para>
      </step>
      
      <step performance="required">
	<para>
	  You should get on the screen (and also written to file ./regress.out)
	  a series of statements stating which tests passed and which tests
	  failed.  Please note that it can be normal for some of the tests to
	  "fail" due to platform-specific variations.  See the next section
	  for details on determining whether a "failure" is significant.
	</para>
	<para>
	  Some of the tests, notably "numeric", can take a while, especially
	  on slower platforms.  Have patience.
	</para>
      </step>
      
      <step performance="required">
	<para>
	  After running the tests and examining the results, type
	  <programlisting>
	    cd /usr/src/pgsql/src/test/regress
	    gmake clean
	  </programlisting>
	  to recover the temporary disk space used by the tests.
	  If you ran a sequential test, also type
	  <programlisting>
	    dropdb regression
	  </programlisting>
	</para>
      </step>
    </procedure>
  </sect1>
  
  <sect1 id="regress-analysis">
    <title>Regression Analysis</title>

     <para>
       The actual outputs of the regression tests are in files in the
       <filename>./results</filename> directory. The test script
       uses <application>diff</application> to compare each output file
       against the reference outputs stored in the
       <filename>./expected</filename> directory.  Any differences are
       saved for your inspection in
       <filename>./regression.diffs</filename>.  (Or you can run
       <application>diff</application> yourself, if you prefer.)
     </para>

     <para>
       The files might not compare exactly.  The test script will report
       any difference as a "failure", but the difference might be due
       to small cross-system differences in error message wording,
       math library behavior, etc.
      "Failures" of this type do not indicate a problem with
      <productname>Postgres</productname>.
d186 1
a186 1
    
d188 4
a191 4
      Thus, it is necessary to examine the actual differences for each
      "failed" test to determine whether there is really a problem.
      The following paragraphs attempt to provide some guidance in
      determining whether a difference is significant or not.
a192 3
    
    <sect2>
      <title>Error message differences</title>
d194 9
a202 10
      <para>
	Some of the regression tests involve intentional invalid input values.
	Error messages can come from either the Postgres code or from the host
	platform system routines. In the latter case, the messages may vary
	between platforms, but should reflect similar information. These
	differences in messages will result in a "failed" regression test which
	can be validated by inspection.
      </para>
      
    </sect2>
d204 2
a205 2
    <sect2>
      <title>Date and time differences</title>
d207 19
a225 31
      <para>
  Most of the date and time results are dependent on timezone environment.
  The reference files are generated for timezone PST8PDT (Berkeley,
  California) and there will be apparent failures if the tests are not
  run with that timezone setting.  The regression test driver sets
  environment variable PGTZ to PST8PDT to ensure proper results.
      </para>

      <para>
       Some of the queries in the "timestamp" test will fail if you run
       the test on the day of a daylight-savings time changeover, or the
       day before or after one.  These queries assume that the intervals
       between midnight yesterday, midnight today and midnight tomorrow are
       exactly twenty-four hours ... which is wrong if daylight-savings time
       went into or out of effect meanwhile.
      </para>

      <para>
  There appear to be some systems which do not accept the recommended syntax
  for explicitly setting the local time zone rules; you may need to use
  a different PGTZ setting on such machines.
      </para>

      <para>
  Some systems using older timezone libraries fail to apply daylight-savings
  corrections to pre-1970 dates, causing pre-1970 PDT times to be displayed
  in PST instead.  This will result in localized differences in the test
  results.
      </para>
      
    </sect2>
d227 2
a228 2
    <sect2>
      <title>Floating point differences</title>
d230 22
a251 17
      <para>
	Some of the tests involve computing 64-bit (<type>float8</type>) numbers from table
	columns. Differences in results involving mathematical functions of
	<type>float8</type> columns have been observed.  The float8
	and geometry tests are particularly prone to small differences
	across platforms.
	Human eyeball comparison is needed to determine the real significance
	of these differences which are usually 10 places to the right of
	the decimal point.
      </para>

      <para>
	Some systems signal errors from pow() and exp() differently from
	the mechanism expected by the current Postgres code.
      </para>
      
    </sect2>
d253 2
a254 2
    <sect2>
      <title>Polygon differences</title>
d256 16
a271 66
      <para>
	Several of the tests involve operations on geographic date about the
	Oakland/Berkley CA street map. The map data is expressed as polygons
	whose vertices are represented as pairs of <type>float8</type> numbers (decimal
	latitude and longitude). Initially, some tables are created and
	loaded with geographic data, then some views are created which join
	two tables using the polygon intersection operator (##), then a select
	is done on the view. 
	
	When comparing the results from different platforms, differences occur
	in the 2nd or 3rd place to the right of the decimal point. The SQL
	statements where these problems occur are the following:
	
	<programlisting>
	  QUERY: SELECT * from street;
	  QUERY: SELECT * from iexit;
	</programlisting>
      </para>
      
    </sect2>
    
    <sect2>
      <title>Random differences</title>
      
      <para>
	There is at least one case in the "random" test script that is
	intended to produce
	random results. This causes random to fail the regression test
	once in a while (perhaps once in every five to ten trials).
	Typing
	<programlisting>
	  diff results/random.out expected/random.out
	</programlisting>
	should produce only one or a few lines of differences.  You need
	not worry unless the random test always fails in repeated attempts.
	(On the other hand, if the random test is <emphasis>never</emphasis>
	reported to fail even in many trials of the regress tests, you
	probably <emphasis>should</emphasis> worry.)
      </para>
      
    </sect2>
    
    <sect2>
      <title>The "expected" files</title>
      
      <para>
	The <filename>./expected/*.out</filename> files were adapted from the original monolithic
	<filename>expected.input</filename> file provided by Jolly Chen et al. Newer versions of these
	files generated on various development machines have been substituted after
	careful (?) inspection. Many of the development machines are running a
	Unix OS variant (FreeBSD, Linux, etc) on Ix86 hardware.
	
	The original <filename>expected.input</filename> file was created on a SPARC Solaris 2.4
	system using the <filename>postgres5-1.02a5.tar.gz</filename> source tree. It was compared
	with a file created on an I386 Solaris 2.4 system and the differences
	were only in the floating point polygons in the 3rd digit to the right
	of the decimal point.
	
	The original <filename>sample.regress.out</filename> file was from the postgres-1.01 release
	constructed by Jolly Chen. It may
	have been created on a DEC ALPHA machine as the <filename>Makefile.global</filename>
	in the postgres-1.01 release has PORTNAME=alpha.
      </para>
      
    </sect2>
    
d274 1
@


1.9
log
@Fixups in content and markup for 7.0 release.
@
text
@d48 1
a48 1
  <sect1>
d194 1
a194 1
  <sect1>
d308 1
a308 1
  <sect1>
d479 1
a479 1
  <sect1>
@


1.8
log
@Mention DST boundaries as a possible cause of regress test failure.
@
text
@d1 2
a2 2
<Chapter Id="regress">
<Title id="regress-title">Regression Test</Title>
d4 188
a191 110
<Abstract>
<Para>
Regression test instructions and analysis.
</Para>
</Abstract>

<Para>
  The PostgreSQL regression tests are a comprehensive set of tests for the
  SQL implementation embedded in PostgreSQL.  They test standard SQL
  operations as well as the extended capabilities of PostgreSQL.
</Para>

<Para>
  There are two different ways in which the regression tests can be run:
  the "sequential" method and the "parallel" method.  The sequential method
  runs each test script in turn, whereas the parallel method starts up
  multiple server processes to run groups of tests in parallel.  Parallel
  testing gives confidence that interprocess communication and locking
  are working correctly.  Another key difference is that the sequential
  test procedure uses an already-installed postmaster, whereas the
  parallel test procedure tests a system that has been built but not yet
  installed.  (The parallel test script actually does an installation into
  a temporary directory and fires up a private postmaster therein.)
</Para>

<Para>
  Some properly installed and fully functional PostgreSQL installations
  can "fail" some of these regression tests due to artifacts of floating point
  representation and time zone support. The tests are currently evaluated
  using a simple <application>diff</application> comparison against the
  outputs generated on a reference system, so the results are sensitive to
  small system differences.
  When a test is reported as "failed", always examine the differences
  between expected and actual results; you may well find that the differences
  are not significant.
</Para>

<Para>
  The regression tests were originally developed by Jolly Chen and Andrew Yu,
  and were extensively revised/repackaged by Marc Fournier and Thomas Lockhart.
  From <ProductName>PostgreSQL</ProductName> v6.1 onward
  the regression tests are current for every official release. 
</Para>

<Sect1>
<Title>Regression Environment</Title>

<Para>
The regression testing notes below assume the following (except where noted):
<ItemizedList Mark="bullet" Spacing="compact">
<ListItem>
<Para>
Commands are Unix-compatible. See note below.
</Para>
</ListItem>
<ListItem>
<Para>
Defaults are used except where noted.
</Para>
</ListItem>
<ListItem>
<Para>
User postgres is the <ProductName>Postgres</ProductName> superuser.
</Para>
</ListItem>
<ListItem>
<Para>
The source path is /usr/src/pgsql (other paths are possible).
</Para>
</ListItem>
<ListItem>
<Para>
The runtime path is /usr/local/pgsql (other paths are possible).
</Para>
</ListItem>
</ItemizedList>
</Para>

<Para>
  Normally, the regression tests should be run as the postgres user since
  the 'src/test/regress' directory and sub-directories are owned by the
  postgres user. If you run the regression test as another user the
  'src/test/regress' directory tree must be writeable by that user.
</Para>

<Para>
  It was formerly necessary to run the postmaster with system time zone
  set to PST, but this is no longer required.  You can run the regression
  tests under your normal postmaster configuration.  The test script will
  set the PGTZ environment variable to ensure that timezone-dependent tests
  produce the expected results.  However, your system must provide
  library support for the PST8PDT time zone, or the timezone-dependent
  tests will fail.
  To verify that your machine does have this support, type
  the following:
<ProgramListing>
    setenv TZ PST8PDT
    date
</ProgramListing>
</Para>

<Para>
  The "date" command above should have returned the current system time
  in the PST8PDT time zone. If the PST8PDT database is not available, then
  your system may have returned the time in GMT. If the PST8PDT time zone
  is not available, you can set the time zone rules explicitly:
<ProgramListing>
    setenv PGTZ PST8PDT7,M04.01.0,M10.05.03
      </ProgramListing>
    </Para>
d194 2
a195 2
  <Sect1>
    <Title>Directory Layout</Title>
d197 1
a197 33
    <Para>
      <Note>
	<Para>
	  This should become a table in the previous section.
	</Para>
      </Note>
    </Para>
    
    <Para>
      <ProgramListing>
  input/ .... .source files that are converted using 'make all' into
              some of the .sql files in the 'sql' subdirectory

  output/ ... .source files that are converted using 'make all' into
              .out files in the 'expected' subdirectory

  sql/ ...... .sql files used to perform the regression tests

  expected/ . .out files that represent what we *expect* the results to
              look like

  results/ .. .out files that contain what the results *actually* look
              like. Also used as temporary storage for table copy testing.

  tmp_check/  temporary installation created by parallel testing script.
      </ProgramListing>
    </Para>
  </Sect1>
  
  <Sect1>
    <Title>Regression Test Procedure</Title>
    
    <Para>
d200 4
a203 3
      like <FileName>ps</FileName> and <FileName>tar</FileName> vary wildly on what options you should use on each
      platform. <Emphasis>Use common sense</Emphasis> before typing in these commands.
    </Para>
d205 2
a206 2
    <Procedure>
      <Title><ProductName>Postgres</ProductName> Regression Test</Title>
d208 2
a209 2
      <Step Performance="required">
	<Para>
d211 1
a211 1
	  <ProgramListing>
d215 1
a215 1
	  </ProgramListing>
d219 2
a220 2
	<Para>
	  This step compiles a <Acronym>C</Acronym>
d225 1
a225 1
	</Para>
d228 2
a229 2
      <Step Performance="optional">
	<Para>
d234 1
a234 1
	  <ProgramListing>
d236 1
a236 1
	  </ProgramListing>
d238 1
a238 1
	  <ProgramListing>
d241 1
a241 1
	  </ProgramListing>
d244 1
a244 1
	  <ProductName>Postgres</ProductName> 7.0) and you might want to
d247 7
a253 7
	  <Note>
	    <Para>
	      Do not run <FileName>postmaster</FileName> from the root account.
	    </Para>
	  </Note>
	</Para>
      </Step>
d255 2
a256 2
      <Step Performance="required">
	<Para>
d258 1
a258 1
	  <ProgramListing>
d261 1
a261 1
	  </ProgramListing>
d263 1
a263 1
	  <ProgramListing>
d266 1
a266 1
	  </ProgramListing>
d270 1
a270 1
	  <ProductName>Postgres</ProductName> into a temporary directory,
d274 2
a275 2
	</Para>
      </Step>
d277 2
a278 2
      <Step Performance="required">
	<Para>
d284 2
a285 2
	</Para>
	<Para>
d288 2
a289 2
	</Para>
      </Step>
d291 2
a292 2
      <Step Performance="required">
	<Para>
d294 1
a294 1
	  <ProgramListing>
d297 1
a297 1
	  </ProgramListing>
d300 1
a300 1
	  <ProgramListing>
d302 3
a304 3
	  </ProgramListing>
	</Para>
      </Step>
d306 1
a306 1
  </Sect1>
d308 2
a309 2
  <Sect1>
    <Title>Regression Analysis</Title>
d311 1
a311 1
     <Para>
d320 1
a320 1
     </Para>
d322 1
a322 1
     <Para>
d328 2
a329 2
      <ProductName>Postgres</ProductName>.
    </Para>
d331 1
a331 1
    <Para>
d336 1
a336 1
    </Para>
d338 2
a339 2
    <Sect2>
      <Title>Error message differences</Title>
d341 1
a341 1
      <Para>
d348 1
a348 1
      </Para>
d350 1
a350 1
    </Sect2>
d352 2
a353 2
    <Sect2>
      <Title>Date and time differences</Title>
d355 1
a355 1
      <Para>
d361 1
a361 1
      </Para>
d363 1
a363 1
      <Para>
d370 1
a370 1
      </Para>
d372 1
a372 1
      <Para>
d376 1
a376 1
      </Para>
d378 1
a378 1
      <Para>
d383 1
a383 1
      </Para>
d385 1
a385 1
    </Sect2>
d387 2
a388 2
    <Sect2>
      <Title>Floating point differences</Title>
d390 2
a391 2
      <Para>
	Some of the tests involve computing 64-bit (<Type>float8</Type>) numbers from table
d393 1
a393 1
	<Type>float8</Type> columns have been observed.  The float8
d399 1
a399 1
      </Para>
d401 1
a401 1
      <Para>
d404 1
a404 1
      </Para>
d406 1
a406 1
    </Sect2>
d408 2
a409 2
    <Sect2>
      <Title>Polygon differences</Title>
d411 1
a411 1
      <Para>
d414 1
a414 1
	whose vertices are represented as pairs of <Type>float8</Type> numbers (decimal
d424 1
a424 1
	<ProgramListing>
d427 2
a428 2
	</ProgramListing>
      </Para>
d430 1
a430 1
    </Sect2>
d432 2
a433 2
    <Sect2>
      <Title>Random differences</Title>
d435 1
a435 1
      <Para>
d441 1
a441 1
	<ProgramListing>
d443 1
a443 1
	</ProgramListing>
d449 1
a449 1
      </Para>
d451 1
a451 1
    </Sect2>
d453 2
a454 2
    <Sect2>
      <Title>The <Quote>expected</Quote> files</Title>
d456 3
a458 3
      <Para>
	The <FileName>./expected/*.out</FileName> files were adapted from the original monolithic
	<FileName>expected.input</FileName> file provided by Jolly Chen et al. Newer versions of these
d463 2
a464 2
	The original <FileName>expected.input</FileName> file was created on a SPARC Solaris 2.4
	system using the <FileName>postgres5-1.02a5.tar.gz</FileName> source tree. It was compared
d469 1
a469 1
	The original <FileName>sample.regress.out</FileName> file was from the postgres-1.01 release
d471 1
a471 1
	have been created on a DEC ALPHA machine as the <FileName>Makefile.global</FileName>
d473 1
a473 1
      </Para>
d475 1
a475 1
    </Sect2>
d477 1
a477 1
  </Sect1>
d479 2
a480 2
  <Sect1>
    <Title>Platform-specific comparison files</Title>
d482 1
a482 1
     <Para>
d491 1
a491 1
     </Para>
d493 1
a493 1
     <Para>
d495 1
a495 1
	<ProgramListing>
d497 1
a497 1
	</ProgramListing>
d503 1
a503 1
     </Para>
d505 1
a505 1
     <Para>
d509 1
a509 1
	<ProgramListing>
d511 1
a511 1
	</ProgramListing>
d513 1
a513 1
	<ProgramListing>
d515 1
a515 1
	</ProgramListing>
d519 1
a519 1
	<ProgramListing>
d521 1
a521 1
	</ProgramListing>
d525 1
a525 1
     </Para>
d527 1
a527 1
  </Sect1>
d529 18
a546 1
</Chapter>
@


1.7
log
@Add description of new parallel regression testing method; update
procedure to cover running either test script; other minor improvements.
@
text
@d317 9
@


1.6
log
@Update platform-specific-expected-file support so that platforms can be
specified by regular-expression patterns.  Add some more files.
@
text
@d2 1
a2 1
<Title>Regression Test</Title>
d17 25
d48 2
a49 8
<Para>
  Some properly installed and fully functional PostgreSQL installations
  can fail some of these regression tests due to artifacts of floating point
  representation and time zone support. The current tests are evaluated
  using a simple "diff" algorithm, and are sensitive to small system
  differences. For apparently failed tests, examining the differences
  may reveal that the differences are not significant.
</Para>
a81 11
<Sect1>
<Title>Regression Environment</Title>

<Para>
  To prepare for regression testing, do <Command>make all</Command> in the regression test
  directory.  This compiles a <Acronym>C</Acronym> program with PostgreSQL extension functions
  into a shared library.  Localized SQL scripts and output-comparison
  files are also created for the tests that need them.  The localization
  replaces macros in the source files with absolute pathnames and user names.  
</Para>

d86 1
a86 1
  'src/test/regress' directory tree must be writeable to that user.
d142 2
a156 5
      
      <Para>
	For a fresh install or upgrading from previous releases of
	<ProductName>Postgres</ProductName>:
      </Para>
d159 2
a160 2
      <Title><ProductName>Postgres</ProductName> Regression Configuration</Title>

d163 15
a177 3
	  The file /usr/src/pgsql/src/test/regress/README has detailed
	  instructions for running and interpreting the regression tests.
	  A short version follows here:
d179 3
a181 1
	
d183 4
a186 2
	  If the postmaster is not already running, start the postmaster on an
	  available window by typing
a189 1
	  
d195 4
a198 5
	</Para>
	
	<Para>
	  Run postmaster from your <ProductName>Postgres</ProductName> super user account (typically
	  account postgres).
a207 17
      <Step Performance="optional">
	<Para>
	  If you have previously invoked the regression test, clean up the
	  working directory with:
	  
	  <ProgramListing>
	    cd /usr/src/pgsql/src/test/regress
	    gmake clean
	  </ProgramListing>
	</para>
	<Para>
	  You do not need to type "gmake clean" if this is the first time you
	  are running the tests.
	</Para>
      </step>

      
d210 1
a210 1
	  Build the regression test. Type
d213 1
a213 1
	    gmake all
d215 1
a215 6
	</Para>
      </Step>
      
      <Step Performance="required">
	<Para>
	  Run the regression tests.  Type
d218 1
a218 1
	    gmake runtest
d220 7
a231 1
	  
d235 6
a240 7
	  "fail".  For the failed tests, use diff to compare the files in
	  directories ./results and ./expected.  If float8 failed, type
	  something like:
	  <ProgramListing>
	    cd /usr/src/pgsql/src/test/regress
	    diff -w expected/float8.out results
	  </ProgramListing>
a247 1
	    dropdb regression
d252 4
d265 8
a272 4
       The results are in files in the ./results directory. These results
       can be compared with results in the ./expected directory using 'diff'.
       (The test script does this for you, and leaves the differences
       in ./regression.diffs.)
d366 1
a366 1
	statements where these problems occur are the folowing:
d380 4
a383 3
	There is at least one test case in random.out which is intended to produce
	random results. This causes random to fail the regression testing
	once in a while.
d388 5
a392 5
	
	should produce only
	one or a few lines of differences for this reason, but other floating
	point differences on dissimilar architectures might cause many more
	differences. See the release notes below.
d411 1
a411 1
	of the decimal point. (see below)
d414 1
a414 1
	constructed by Jolly Chen and is included here for reference. It may
@


1.5
log
@Okay, that should put us back in sync. These two patches (src & doc) are
against the sources from one hour ago and contain all the portable and
up
to date stuff.

A few other CVS "householding" things you might want to take care of:

* Remove the src/bin/cleardbdir directory

* Remove the file src/bin/psql/sql_help.h from the repository, as it is
a derived file and is build by the release_prep.

Peter Eisentraut
@
text
@d12 2
a13 3
  SQL implementation embedded in PostgreSQL developed by Jolly Chen and
  Andrew Yu. It tests standard SQL operations as well as the extended
  capabilities of PostgreSQL.
d17 4
a20 5
  These tests have recently been revised by Marc Fournier and Thomas Lockhart
and are now packaged as
  functional units which should make them easier to run and easier to interpret.
From <ProductName>PostgreSQL</ProductName> v6.1 onward
 the regression tests are current for every official release. 
d67 5
a71 6
  The regression test is invoked by the <Command>make</Command> command which compiles
  a <Acronym>C</Acronym> program into a shared library
  in the current directory.  Localized shell scripts are also created in
  the current directory. The output file templates are massaged into the
  <FileName>./expected/*.out</FileName> files.  The localization replaces macros in the source
  files with absolute pathnames and user names.
d75 1
a75 1
  Normally, the regression test should be run as the pg_superuser since
d77 2
a78 2
  pg_superuser. If you run the regression test as another user the
  'src/test/regress' directory tree should be writeable to that user.
d132 1
a132 1
  results/ .. .out files that represent what the results *actually* look
a296 19
      <Title>OID differences</Title>
      
      <Para>
	There are several places where PostgreSQL OID (object identifiers) appear
	in 'regress.out'. OID's are unique 32-bit integers which are generated
	by the PostgreSQL backend whenever a table row is inserted or updated.
	If you run the regression test on a non-virgin database or run it multiple
	times, the OID's reported will have different values. 
	
	The following SQL statements in 'misc.out' have shown this behavior:
	
	QUERY: SELECT user_relns() AS user_relns ORDER BY user_relns;
	
	The 'a,523676' row is composed from an OID.
      </Para>
      
    </Sect2>
    
    <Sect2>
d410 50
@


1.4
log
@Update SGML docs with some changes I made to test/regress/README
many moons ago, but forgot to put here.
@
text
@d249 1
a249 1
	    destroydb regression
@


1.3
log
@Clean up to ensure tag completion as required by the newest versions
 of Norm's Modular Style Sheets and jade/docbook.
From Vince Vielhaber <vev@@michvhf.com>.
@
text
@d85 9
a93 9
  The postmaster should be invoked with the system time zone set for
  Berkeley, California. This is done automatically by the regression
test script. However, it does require machine support for the PST8PDT
time zone.
</Para>

<Para>
To verify that your machine does have this support, type
the following:
a149 3
    
    <Procedure>
      <Title><ProductName>Postgres</ProductName> Regression Configuration</Title>
d155 4
a158 22
      
      <Step Performance="required">
	<Para>
	  Build the regression test. Type
	  <ProgramListing>
	    cd /usr/src/pgsql/src/test/regress
	    gmake all
	  </ProgramListing>
	</Para>
      </Step>
      
      <Step Performance="optional">
	<Para>
	  If you have prevously invoked the regression test, clean up the
	  working directory with:
	  
	  <ProgramListing>
	    cd /usr/src/pgsql/src/test/regress
	    make clean
	  </ProgramListing>
	</para>
      </step>
d192 1
a192 1
      <Step Performance="required">
d194 2
a195 1
	  Run the regression tests.  Type
d199 1
a199 1
	    gmake runtest
d201 1
a201 2
	</Para>
	
a202 1
	  
d206 21
d247 1
a247 1
	  After running the tests, type
d253 1
d261 13
a273 4
    
    <Para>
      <Quote>Failed</Quote> tests may have failed due to slightly different error messages,
      math libraries, or output formatting.
d279 4
a282 19
      For a i686/Linux-ELF platform, no tests failed since this is the
      v6.2.1 regression testing reference platform.
    </Para>
    
    <Para>
      For the SPARC/Linux-ELF platform, using the 970525 beta version of
      <ProductName>Postgres</ProductName> v6.2 the following tests "failed":
      float8 and geometry "failed" due to minor precision differences in
      floating point numbers.  select_views produces massively different output,
      but the differences are due to minor floating point differences.
    </Para>
    
    <Para>
      Conclusion?  If you do see failures, try to understand the nature of
      the differences and then decide if those differences will affect your
      intended use of <ProductName>Postgres</ProductName>.  However, keep in mind that this is likely
      to be the most solid release of <ProductName>Postgres</ProductName> to date, incorporating many
      bug fixes from v6.1, and that previous versions of <ProductName>Postgres</ProductName> have been
      in use successfully for some time now.
a285 12
      <Title>Comparing expected/actual output</Title>

      <Para>
	The results are in files in the ./results directory. These results
	can be compared with results in the ./expected directory using 'diff'.
	The files might not compare exactly. The following paragraphs attempt
	to explain the differences.
      </Para>
      
    </Sect2>
    
    <Sect2>
d322 18
a339 12
	On many supported platforms, you can force PostgreSQL to believe that it
	is running in the same time zone as Berkeley, California. See details in
	the section on how to run the regression tests.
	
	If you do not explicitly set your time zone environment to PST8PDT, then
	most of the date and time results will reflect your local time zone and
	will fail the regression testing.
	
	There appears to be some systems which do not accept the recommended syntax
	for explicitly setting the local time zone rules. Some systems using the
	public domain time zone package exhibit minor problems with pre-1970 PDT
	times, representing them in PST instead.
d348 1
a348 1
	Some of the tests involve computing 64-bit (<Type>float8</Type>) number from table
d350 3
a352 5
	<Type>float8</Type> columns have been observed. These differences occur where
	different operating systems are used on the same platform ie:
	BSDI and SOLARIS on Intel/86, and where the same operating system is
	used used on different platforms, ie: SOLARIS on SPARC and Intel/86.
	
d356 3
a358 1
	
d394 2
a395 1
	random results. This causes random to fail the regression testing.
@


1.2
log
@Add id field to chapter and book tags to allow output file names
 to be meaningful.
@
text
@d107 17
a123 16
</ProgramListing>
</Para>

<Sect1>
<Title>Directory Layout</Title>

<Para>
<Note>
<Para>
This should become a table in the previous section.
</Para>
</Note>
</Para>

<Para>
<ProgramListing>
d137 309
a445 312
</ProgramListing>
</Para>

</Sect1>

<Sect1>
<Title>Regression Test Procedure</Title>

<Para>
Commands were tested on RedHat Linux version 4.2 using the bash shell.
Except where noted, they will probably work on most systems. Commands
like <FileName>ps</FileName> and <FileName>tar</FileName> vary wildly on what options you should use on each
platform. <Emphasis>Use common sense</Emphasis> before typing in these commands.
</Para>

<Para>
<Procedure>
<Title><ProductName>Postgres</ProductName> Regression Configuration</Title>

<Para>
For a fresh install or upgrading from previous releases of
<ProductName>Postgres</ProductName>:
</Para>

<Step Performance="required">
<Para>
Build the regression test. Type
<ProgramListing>
    cd /usr/src/pgsql/src/test/regress
    gmake all
</ProgramListing>
</Para>
</Step>

<Step Performance="optional">
<Para>
  If you have prevously invoked the regression test, clean up the
  working directory with:

<ProgramListing>
    cd /usr/src/pgsql/src/test/regress
    make clean
</ProgramListing>

<Step Performance="required">
<Para>
     The file /usr/src/pgsql/src/test/regress/README has detailed
     instructions for running and interpreting the regression tests.
     A short version follows here:
</Para>

<Para>
If the postmaster is not already running, start the postmaster on an
available window by typing
<ProgramListing>
    postmaster
</ProgramListing>

or start the postmaster daemon running in the background by typing
<ProgramListing>
    cd
    nohup postmaster > regress.log 2>&1 &
</ProgramListing>
</Para>

<Para>
     Run postmaster from your <ProductName>Postgres</ProductName> super user account (typically
     account postgres).

<Note>
<Para>
Do not run <FileName>postmaster</FileName> from the root account.
</Para>
</Note>
</Para>
</Step>

<Step Performance="required">
<Para>
 Run the regression tests.  Type

<ProgramListing>
    cd /usr/src/pgsql/src/test/regress
    gmake runtest
</ProgramListing>
</Para>

<Para>

     You do not need to type "gmake clean" if this is the first time you
     are running the tests.
</Para>
</Step>

<Step Performance="required">
<Para>

     You should get on the screen (and also written to file ./regress.out)
     a series of statements stating which tests passed and which tests
     failed.  Please note that it can be normal for some of the tests to
     "fail".  For the failed tests, use diff to compare the files in
     directories ./results and ./expected.  If float8 failed, type
     something like:
<ProgramListing>
    cd /usr/src/pgsql/src/test/regress
    diff -w expected/float8.out results
</ProgramListing>
</Para>
</Step>

<Step Performance="required">
<Para>
     After running the tests, type
<ProgramListing>
    destroydb regression
    cd /usr/src/pgsql/src/test/regress
    gmake clean
</ProgramListing>
</Para>
</Step>

</Procedure>

</Sect1>

<Sect1>
<Title>Regression Analysis</Title>

<Para>
    <Quote>Failed</Quote> tests may have failed due to slightly different error messages,
     math libraries, or output formatting.
     "Failures" of this type do not indicate a problem with
     <ProductName>Postgres</ProductName>.
</Para>

<Para>
     For a i686/Linux-ELF platform, no tests failed since this is the
     v6.2.1 regression testing reference platform.
</Para>

<Para>
     For the SPARC/Linux-ELF platform, using the 970525 beta version of
     <ProductName>Postgres</ProductName> v6.2 the following tests "failed":
     float8 and geometry "failed" due to minor precision differences in
     floating point numbers.  select_views produces massively different output,
     but the differences are due to minor floating point differences.
</Para>

<Para>
     Conclusion?  If you do see failures, try to understand the nature of
     the differences and then decide if those differences will affect your
     intended use of <ProductName>Postgres</ProductName>.  However, keep in mind that this is likely
     to be the most solid release of <ProductName>Postgres</ProductName> to date, incorporating many
     bug fixes from v6.1, and that previous versions of <ProductName>Postgres</ProductName> have been
     in use successfully for some time now.
</Para>

<Sect2>
<Title>Comparing expected/actual output</Title>

<Para>
  The results are in files in the ./results directory. These results
  can be compared with results in the ./expected directory using 'diff'.
  The files might not compare exactly. The following paragraphs attempt
  to explain the differences.
</Para>

</Sect2>

<Sect2>
<Title>Error message differences</Title>

<Para>
  Some of the regression tests involve intentional invalid input values.
  Error messages can come from either the Postgres code or from the host
  platform system routines. In the latter case, the messages may vary
  between platforms, but should reflect similar information. These
  differences in messages will result in a "failed" regression test which
  can be validated by inspection.
</Para>

</Sect2>

<Sect2>
<Title>OID differences</Title>

<Para>
  There are several places where PostgreSQL OID (object identifiers) appear
  in 'regress.out'. OID's are unique 32-bit integers which are generated
  by the PostgreSQL backend whenever a table row is inserted or updated.
  If you run the regression test on a non-virgin database or run it multiple
  times, the OID's reported will have different values. 

  The following SQL statements in 'misc.out' have shown this behavior:

  QUERY: SELECT user_relns() AS user_relns ORDER BY user_relns;

    The 'a,523676' row is composed from an OID.
</Para>

</Sect2>

<Sect2>
<Title>Date and time differences</Title>

<Para>
  On many supported platforms, you can force PostgreSQL to believe that it
  is running in the same time zone as Berkeley, California. See details in
  the section on how to run the regression tests.

  If you do not explicitly set your time zone environment to PST8PDT, then
  most of the date and time results will reflect your local time zone and
  will fail the regression testing.

  There appears to be some systems which do not accept the recommended syntax
  for explicitly setting the local time zone rules. Some systems using the
  public domain time zone package exhibit minor problems with pre-1970 PDT
  times, representing them in PST instead.
</Para>

</Sect2>

<Sect2>
<Title>Floating point differences</Title>

<Para>
  Some of the tests involve computing 64-bit (<Type>float8</Type>) number from table
  columns. Differences in results involving mathematical functions of
  <Type>float8</Type> columns have been observed. These differences occur where
  different operating systems are used on the same platform ie:
  BSDI and SOLARIS on Intel/86, and where the same operating system is
  used used on different platforms, ie: SOLARIS on SPARC and Intel/86.

  Human eyeball comparison is needed to determine the real significance
  of these differences which are usually 10 places to the right of
  the decimal point.

  Some systems signal errors from pow() and exp() differently from
  the mechanism expected by the current Postgres code.
</Para>

</Sect2>

<Sect2>
<Title>Polygon differences</Title>

<Para>
  Several of the tests involve operations on geographic date about the
  Oakland/Berkley CA street map. The map data is expressed as polygons
  whose vertices are represented as pairs of <Type>float8</Type> numbers (decimal
  latitude and longitude). Initially, some tables are created and
  loaded with geographic data, then some views are created which join
  two tables using the polygon intersection operator (##), then a select
  is done on the view. 

  When comparing the results from different platforms, differences occur
  in the 2nd or 3rd place to the right of the decimal point. The SQL
  statements where these problems occur are the folowing:

<ProgramListing>
    QUERY: SELECT * from street;
    QUERY: SELECT * from iexit;
</ProgramListing>
</Para>

</Sect2>

<Sect2>
<Title>Random differences</Title>

<Para>
  There is at least one test case in random.out which is intended to produce
  random results. This causes random to fail the regression testing.
  Typing
<ProgramListing>
    diff results/random.out expected/random.out
</ProgramListing>

 should produce only
  one or a few lines of differences for this reason, but other floating
  point differences on dissimilar architectures might cause many more
  differences. See the release notes below.
</Para>

</Sect2>

<Sect2>
<Title>The <Quote>expected</Quote> files</Title>

<Para>
  The <FileName>./expected/*.out</FileName> files were adapted from the original monolithic
  <FileName>expected.input</FileName> file provided by Jolly Chen et al. Newer versions of these
  files generated on various development machines have been substituted after
  careful (?) inspection. Many of the development machines are running a
  Unix OS variant (FreeBSD, Linux, etc) on Ix86 hardware.

  The original <FileName>expected.input</FileName> file was created on a SPARC Solaris 2.4
  system using the <FileName>postgres5-1.02a5.tar.gz</FileName> source tree. It was compared
  with a file created on an I386 Solaris 2.4 system and the differences
  were only in the floating point polygons in the 3rd digit to the right
  of the decimal point. (see below)

  The original <FileName>sample.regress.out</FileName> file was from the postgres-1.01 release
  constructed by Jolly Chen and is included here for reference. It may
  have been created on a DEC ALPHA machine as the <FileName>Makefile.global</FileName>
  in the postgres-1.01 release has PORTNAME=alpha.
</Para>

</Sect2>

</Sect1>

@


1.1
log
@SGML source for new documentation.
@
text
@d1 1
a1 1
<Chapter>
@
