head	1.43;
access;
symbols
	REL9_0_0:1.43
	REL9_1_ALPHA1:1.43
	REL9_0_RC1:1.43
	REL9_0_BETA4:1.43
	REL9_0_STABLE:1.43.0.2
	REL9_0_BETA3:1.43
	REL9_0_BETA2:1.43
	REL7_4_29:1.24
	REL8_0_25:1.27.4.1
	REL8_1_21:1.33
	REL8_2_17:1.38
	REL8_3_11:1.40
	REL8_4_4:1.40.4.1
	REL9_0_BETA1:1.43
	REL9_0_ALPHA5_BRANCH:1.42.0.10
	REL9_0_ALPHA5:1.42
	REL7_4_28:1.24
	REL8_0_24:1.27.4.1
	REL8_1_20:1.33
	REL8_2_16:1.38
	REL8_3_10:1.40
	REL8_4_3:1.40.4.1
	REL9_0_ALPHA4:1.42
	REL9_0_ALPHA4_BRANCH:1.42.0.8
	REL8_5_ALPHA3:1.42
	REL8_5_ALPHA3_BRANCH:1.42.0.6
	REL7_4_27:1.24
	REL8_0_23:1.27.4.1
	REL8_1_19:1.33
	REL8_2_15:1.38
	REL8_3_9:1.40
	REL8_4_2:1.40.4.1
	REL8_5_ALPHA2:1.42
	REL8_5_ALPHA2_BRANCH:1.42.0.4
	REL7_4_26:1.24
	REL8_0_22:1.27.4.1
	REL8_1_18:1.33
	REL8_2_14:1.38
	REL8_3_8:1.40
	REL8_4_1:1.40.4.1
	REL8_5_ALPHA1:1.42
	REL8_5_ALPHA1_BRANCH:1.42.0.2
	REL8_4_STABLE:1.40.0.4
	REL8_4_0:1.40
	REL8_4_RC2:1.40
	REL8_4_RC1:1.40
	REL8_4_BETA2:1.40
	REL8_4_BETA1:1.40
	REL7_4_25:1.24
	REL8_0_21:1.27.4.1
	REL8_1_17:1.33
	REL8_2_13:1.38
	REL8_3_7:1.40
	REL7_4_24:1.24
	REL8_0_20:1.27.4.1
	REL8_1_16:1.33
	REL8_2_12:1.38
	REL8_3_6:1.40
	REL7_4_23:1.24
	REL8_0_19:1.27.4.1
	REL8_1_15:1.33
	REL8_2_11:1.38
	REL8_3_5:1.40
	REL7_4_22:1.24
	REL8_0_18:1.27.4.1
	REL8_1_14:1.33
	REL8_2_10:1.38
	REL8_3_4:1.40
	REL7_4_21:1.24
	REL8_0_17:1.27.4.1
	REL8_1_13:1.33
	REL8_2_9:1.38
	REL8_3_3:1.40
	REL7_4_20:1.24
	REL8_0_16:1.27.4.1
	REL8_1_12:1.33
	REL8_2_8:1.38
	REL8_3_2:1.40
	REL8_2_7:1.38
	REL8_3_1:1.40
	REL8_3_STABLE:1.40.0.2
	REL8_3_0:1.40
	REL8_3_RC2:1.40
	REL7_3_21:1.23
	REL7_4_19:1.24
	REL8_0_15:1.27.4.1
	REL8_1_11:1.33
	REL8_2_6:1.38
	REL8_3_RC1:1.40
	REL8_3_BETA4:1.40
	REL8_3_BETA3:1.40
	REL8_3_BETA2:1.40
	REL8_3_BETA1:1.40
	REL7_3_20:1.23
	REL7_4_18:1.24
	REL8_0_14:1.27.4.1
	REL8_1_10:1.33
	REL8_2_5:1.38
	REL7_3_19:1.23
	REL7_4_17:1.24
	REL8_0_13:1.27.4.1
	REL8_1_9:1.33
	REL8_2_4:1.38
	REL8_0_12:1.27.4.1
	REL8_1_8:1.33
	REL8_2_3:1.38
	REL7_3_18:1.23
	REL7_4_16:1.24
	REL8_0_11:1.27.4.1
	REL8_1_7:1.33
	REL8_2_2:1.38
	REL8_0_10:1.27.4.1
	REL8_1_6:1.33
	REL8_2_1:1.38
	REL7_4_15:1.24
	REL7_3_17:1.23
	REL8_2_STABLE:1.38.0.2
	REL8_2_0:1.38
	REL8_2_RC1:1.38
	REL8_2_BETA3:1.38
	REL8_2_BETA2:1.38
	REL8_1_5:1.33
	REL8_0_9:1.27.4.1
	REL7_4_14:1.24
	REL7_3_16:1.23
	REL8_2_BETA1:1.38
	REL7_3_15:1.23
	REL7_4_13:1.24
	REL8_0_8:1.27.4.1
	REL8_1_4:1.33
	REL7_3_14:1.23
	REL7_4_12:1.24
	REL8_0_7:1.27.4.1
	REL8_1_3:1.33
	REL7_3_13:1.23
	REL7_4_11:1.24
	REL8_0_6:1.27.4.1
	REL8_1_2:1.33
	REL7_3_12:1.23
	REL7_4_10:1.24
	REL8_0_5:1.27.4.1
	REL8_1_1:1.33
	REL8_1_STABLE:1.33.0.2
	REL8_1_0:1.33
	REL8_1_0RC1:1.33
	REL8_1_0BETA4:1.32
	REL8_1_0BETA3:1.32
	REL7_3_11:1.23
	REL7_4_9:1.24
	REL8_0_4:1.27.4.1
	REL8_1_0BETA2:1.32
	REL8_1_0BETA1:1.32
	REL7_2_8:1.23
	REL7_3_10:1.23
	REL7_4_8:1.24
	REL8_0_3:1.27.4.1
	REL8_0_2:1.27.4.1
	REL7_2_7:1.23
	REL7_3_9:1.23
	REL7_4_7:1.24
	REL8_0_1:1.27.4.1
	REL8_0_STABLE:1.27.0.4
	REL8_0_0:1.27.0.2
	REL8_0_0RC5:1.27
	REL8_0_0RC4:1.27
	REL8_0_0RC3:1.26
	REL8_0_0RC2:1.26
	REL8_0_0RC1:1.26
	REL8_0_0BETA5:1.26
	REL8_0_0BETA4:1.26
	REL7_4_6:1.24
	REL7_3_8:1.23
	REL7_2_6:1.23
	REL8_0_0BETA3:1.26
	REL8_0_0BETA2:1.26
	REL7_2_5:1.23
	REL7_4_5:1.24
	REL7_3_7:1.23
	REL7_4_4:1.24
	REL8_0_0BETA1:1.26
	REL7_4_3:1.24
	REL7_4_2:1.24
	REL7_3_6:1.23
	REL7_4_1:1.24
	REL7_3_5:1.23
	REL7_4:1.24
	REL7_4_RC2:1.24
	REL7_4_STABLE:1.24.0.2
	REL7_4_RC1:1.24
	REL7_4_BETA5:1.24
	REL7_4_BETA4:1.24
	REL7_4_BETA3:1.23
	REL7_4_BETA2:1.23
	WIN32_DEV:1.23.0.6
	REL7_4_BETA1:1.23
	REL7_3_4:1.23
	REL7_3_2:1.23
	REL7_2_4:1.23
	REL7_3_STABLE:1.23.0.4
	REL7_2_3:1.23
	REL7_2_STABLE:1.23.0.2
	REL7_2:1.23
	REL7_2_RC2:1.23
	REL7_2_RC1:1.23
	REL7_2_BETA5:1.22
	REL7_2_BETA4:1.20
	REL7_2_BETA3:1.19
	REL7_2_BETA2:1.19
	REL7_2_BETA1:1.19
	REL7_1_2:1.16
	REL7_1_STABLE:1.16.0.2
	REL7_1_BETA:1.13
	REL7_1_BETA3:1.15
	REL7_1_BETA2:1.15
	REL7_1:1.15
	REL7_0_PATCHES:1.9.0.2
	REL7_0:1.9
	REL6_5_PATCHES:1.8.0.2
	REL6_5:1.8
	REL6_4:1.4.0.2
	release-6-3:1.1;
locks; strict;
comment	@# @;


1.43
date	2010.04.03.07.22.54;	author petere;	state Exp;
branches;
next	1.42;

1.42
date	2009.08.10.22.41.38;	author alvherre;	state Exp;
branches;
next	1.41;

1.41
date	2009.07.16.20.55.44;	author tgl;	state Exp;
branches;
next	1.40;

1.40
date	2007.07.21.04.02.41;	author tgl;	state Exp;
branches
	1.40.4.1;
next	1.39;

1.39
date	2007.02.16.03.50.29;	author momjian;	state Exp;
branches;
next	1.38;

1.38
date	2006.09.16.00.30.14;	author momjian;	state Exp;
branches;
next	1.37;

1.37
date	2006.09.14.11.16.27;	author teodor;	state Exp;
branches;
next	1.36;

1.36
date	2006.03.10.19.10.48;	author momjian;	state Exp;
branches;
next	1.35;

1.35
date	2006.01.22.03.56.58;	author neilc;	state Exp;
branches;
next	1.34;

1.34
date	2005.11.07.17.36.44;	author tgl;	state Exp;
branches;
next	1.33;

1.33
date	2005.10.25.13.38.09;	author momjian;	state Exp;
branches;
next	1.32;

1.32
date	2005.04.12.03.16.50;	author tgl;	state Exp;
branches;
next	1.31;

1.31
date	2005.04.09.03.52.43;	author momjian;	state Exp;
branches;
next	1.30;

1.30
date	2005.03.31.03.54.38;	author momjian;	state Exp;
branches;
next	1.29;

1.29
date	2005.02.21.02.21.00;	author neilc;	state Exp;
branches;
next	1.28;

1.28
date	2005.01.22.22.56.36;	author momjian;	state Exp;
branches;
next	1.27;

1.27
date	2005.01.05.23.42.03;	author tgl;	state Exp;
branches
	1.27.4.1;
next	1.26;

1.26
date	2003.11.29.19.51.37;	author pgsql;	state Exp;
branches;
next	1.25;

1.25
date	2003.11.24.19.08.02;	author petere;	state Exp;
branches;
next	1.24;

1.24
date	2003.09.29.18.18.35;	author momjian;	state Exp;
branches;
next	1.23;

1.23
date	2002.01.20.22.19.56;	author petere;	state Exp;
branches;
next	1.22;

1.22
date	2002.01.08.18.03.26;	author petere;	state Exp;
branches;
next	1.21;

1.21
date	2002.01.08.05.39.26;	author momjian;	state Exp;
branches;
next	1.20;

1.20
date	2001.11.21.05.53.41;	author thomas;	state Exp;
branches;
next	1.19;

1.19
date	2001.10.09.18.46.00;	author petere;	state Exp;
branches;
next	1.18;

1.18
date	2001.09.13.15.55.22;	author petere;	state Exp;
branches;
next	1.17;

1.17
date	2001.05.17.21.50.15;	author petere;	state Exp;
branches;
next	1.16;

1.16
date	2001.04.20.15.52.33;	author thomas;	state Exp;
branches;
next	1.15;

1.15
date	2000.12.22.21.51.57;	author petere;	state Exp;
branches;
next	1.14;

1.14
date	2000.12.16.22.44.47;	author tgl;	state Exp;
branches;
next	1.13;

1.13
date	2000.09.29.20.21.33;	author petere;	state Exp;
branches;
next	1.12;

1.12
date	2000.09.18.20.11.37;	author petere;	state Exp;
branches;
next	1.11;

1.11
date	2000.08.23.05.59.02;	author thomas;	state Exp;
branches;
next	1.10;

1.10
date	2000.06.28.03.30.53;	author tgl;	state Exp;
branches;
next	1.9;

1.9
date	2000.03.31.03.27.40;	author thomas;	state Exp;
branches;
next	1.8;

1.8
date	99.03.30.15.25.56;	author thomas;	state Exp;
branches;
next	1.7;

1.7
date	99.02.19.01.57.08;	author thomas;	state Exp;
branches;
next	1.6;

1.6
date	99.02.18.05.26.17;	author momjian;	state Exp;
branches;
next	1.5;

1.5
date	98.12.29.02.24.15;	author thomas;	state Exp;
branches;
next	1.4;

1.4
date	98.08.15.06.55.05;	author thomas;	state Exp;
branches;
next	1.3;

1.3
date	98.07.29.06.35.16;	author thomas;	state Exp;
branches;
next	1.2;

1.2
date	98.04.04.16.32.01;	author thomas;	state Exp;
branches;
next	1.1;

1.1
date	98.03.01.08.15.53;	author thomas;	state Exp;
branches;
next	;

1.27.4.1
date	2005.01.22.23.05.47;	author momjian;	state Exp;
branches;
next	;

1.40.4.1
date	2009.08.10.22.42.41;	author alvherre;	state Exp;
branches;
next	;


desc
@@


1.43
log
@Remove unnecessary xref endterm attributes and title ids

The endterm attribute is mainly useful when the toolchain does not support
automatic link target text generation for a particular situation.  In  the
past, this was required by the man page tools for all reference page links,
but that is no longer the case, and it now actually gets in the way of
proper automatic link text generation.  The only remaining use cases are
currently xrefs to refsects.
@
text
@<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.42 2009/08/10 22:41:38 alvherre Exp $ -->

 <chapter id="geqo">
  <chapterinfo>
   <author>
    <firstname>Martin</firstname>
    <surname>Utesch</surname>
    <affiliation>
     <orgname>
      University of Mining and Technology
     </orgname>
     <orgdiv>
      Institute of Automatic Control
     </orgdiv>
     <address>
      <city>
       Freiberg
      </city>
      <country>
       Germany
      </country>
     </address>
    </affiliation>
   </author>
   <date>1997-10-02</date>
  </chapterinfo>

  <title>Genetic Query Optimizer</title>

  <para>
   <note>
    <title>Author</title>
    <para>
     Written by Martin Utesch (<email>utesch@@aut.tu-freiberg.de</email>)
     for the Institute of Automatic Control at the University of Mining and Technology in Freiberg, Germany.
    </para>
   </note>
  </para>

  <sect1 id="geqo-intro">
   <title>Query Handling as a Complex Optimization Problem</title>

   <para>
    Among all relational operators the most difficult one to process
    and optimize is the <firstterm>join</firstterm>. The number of
    possible query plans grows exponentially with the
    number of joins in the query. Further optimization effort is
    caused by the support of a variety of <firstterm>join
    methods</firstterm> (e.g., nested loop, hash join, merge join in
    <productname>PostgreSQL</productname>) to process individual joins
    and a diversity of <firstterm>indexes</firstterm> (e.g.,
    B-tree, hash, GiST and GIN in <productname>PostgreSQL</productname>) as
    access paths for relations.
   </para>

   <para>
    The normal <productname>PostgreSQL</productname> query optimizer
    performs a <firstterm>near-exhaustive search</firstterm> over the
    space of alternative strategies. This algorithm, first introduced
    in IBM's System R database, produces a near-optimal join order,
    but can take an enormous amount of time and memory space when the
    number of joins in the query grows large. This makes the ordinary
    <productname>PostgreSQL</productname> query optimizer
    inappropriate for queries that join a large number of tables.
   </para>

   <para>
    The Institute of Automatic Control at the University of Mining and
    Technology, in Freiberg, Germany, encountered some problems when
    it wanted to use <productname>PostgreSQL</productname> as the
    backend for a decision support knowledge based system for the
    maintenance of an electrical power grid. The DBMS needed to handle
    large join queries for the inference machine of the knowledge
    based system. The number of joins in these queries made using the
    normal query optimizer infeasible.
   </para>

   <para>
    In the following we describe the implementation of a
    <firstterm>genetic algorithm</firstterm> to solve the join
    ordering problem in a manner that is efficient for queries
    involving large numbers of joins.
   </para>
  </sect1>

  <sect1 id="geqo-intro2">
   <title>Genetic Algorithms</title>

   <para>
    The genetic algorithm (<acronym>GA</acronym>) is a heuristic optimization method which
    operates through randomized search. The set of possible solutions for the
    optimization problem is considered as a
    <firstterm>population</firstterm> of <firstterm>individuals</firstterm>.
    The degree of adaptation of an individual to its environment is specified
    by its <firstterm>fitness</firstterm>.
   </para>

   <para>
    The coordinates of an individual in the search space are represented
    by <firstterm>chromosomes</firstterm>, in essence a set of character
    strings. A <firstterm>gene</firstterm> is a
    subsection of a chromosome which encodes the value of a single parameter
    being optimized. Typical encodings for a gene could be <firstterm>binary</firstterm> or
    <firstterm>integer</firstterm>.
   </para>

   <para>
    Through simulation of the evolutionary operations <firstterm>recombination</firstterm>,
    <firstterm>mutation</firstterm>, and
    <firstterm>selection</firstterm> new generations of search points are found
    that show a higher average fitness than their ancestors.
   </para>

   <para>
    According to the <systemitem class="resource">comp.ai.genetic</> <acronym>FAQ</acronym> it cannot be stressed too
    strongly that a <acronym>GA</acronym> is not a pure random search for a solution to a
    problem. A <acronym>GA</acronym> uses stochastic processes, but the result is distinctly
    non-random (better than random).
   </para>

   <figure id="geqo-diagram">
    <title>Structured Diagram of a Genetic Algorithm</title>

    <informaltable frame="none">
     <tgroup cols="2">
      <tbody>
       <row>
        <entry>P(t)</entry>
        <entry>generation of ancestors at a time t</entry>
       </row>

       <row>
        <entry>P''(t)</entry>
        <entry>generation of descendants at a time t</entry>
       </row>
      </tbody>
     </tgroup>
    </informaltable>

<literallayout class="monospaced">
+=========================================+
|&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;  Algorithm GA  &lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;|
+=========================================+
| INITIALIZE t := 0                       |
+=========================================+
| INITIALIZE P(t)                         |
+=========================================+
| evaluate FITNESS of P(t)                |
+=========================================+
| while not STOPPING CRITERION do         |
|   +-------------------------------------+
|   | P'(t)  := RECOMBINATION{P(t)}       |
|   +-------------------------------------+
|   | P''(t) := MUTATION{P'(t)}           |
|   +-------------------------------------+
|   | P(t+1) := SELECTION{P''(t) + P(t)}  |
|   +-------------------------------------+
|   | evaluate FITNESS of P''(t)          |
|   +-------------------------------------+
|   | t := t + 1                          |
+===+=====================================+
</literallayout>
   </figure>
  </sect1>

  <sect1 id="geqo-pg-intro">
   <title>Genetic Query Optimization (<acronym>GEQO</acronym>) in PostgreSQL</title>

   <para>
    The <acronym>GEQO</acronym> module approaches the query
    optimization problem as though it were the well-known traveling salesman
    problem (<acronym>TSP</acronym>).
    Possible query plans are encoded as integer strings. Each string
    represents the join order from one relation of the query to the next.
    For example, the join tree
<literallayout class="monospaced">
   /\
  /\ 2
 /\ 3
4  1
</literallayout>
    is encoded by the integer string '4-1-3-2',
    which means, first join relation '4' and '1', then '3', and
    then '2', where 1, 2, 3, 4 are relation IDs within the
    <productname>PostgreSQL</productname> optimizer.
   </para>

   <para>
    Specific characteristics of the <acronym>GEQO</acronym>
    implementation in <productname>PostgreSQL</productname>
    are:

    <itemizedlist spacing="compact" mark="bullet">
     <listitem>
      <para>
       Usage of a <firstterm>steady state</firstterm> <acronym>GA</acronym> (replacement of the least fit
       individuals in a population, not whole-generational replacement)
       allows fast convergence towards improved query plans. This is
       essential for query handling with reasonable time;
      </para>
     </listitem>

     <listitem>
      <para>
       Usage of <firstterm>edge recombination crossover</firstterm>
       which is especially suited to keep edge losses low for the
       solution of the <acronym>TSP</acronym> by means of a
       <acronym>GA</acronym>;
      </para>
     </listitem>

     <listitem>
      <para>
       Mutation as genetic operator is deprecated so that no repair
       mechanisms are needed to generate legal <acronym>TSP</acronym> tours.
      </para>
     </listitem>
    </itemizedlist>
   </para>

   <para>
    Parts of the <acronym>GEQO</acronym> module are adapted from D. Whitley's
    Genitor algorithm.
   </para>

   <para>
    The <acronym>GEQO</acronym> module allows
    the <productname>PostgreSQL</productname> query optimizer to
    support large join queries effectively through
    non-exhaustive search.
   </para>

  <sect2>
   <title>Generating Possible Plans with <acronym>GEQO</acronym></title>

   <para>
    The <acronym>GEQO</acronym> planning process uses the standard planner
    code to generate plans for scans of individual relations.  Then join
    plans are developed using the genetic approach.  As shown above, each
    candidate join plan is represented by a sequence in which to join
    the base relations.  In the initial stage, the <acronym>GEQO</acronym>
    code simply generates some possible join sequences at random.  For each
    join sequence considered, the standard planner code is invoked to
    estimate the cost of performing the query using that join sequence.
    (For each step of the join sequence, all three possible join strategies
    are considered; and all the initially-determined relation scan plans
    are available.  The estimated cost is the cheapest of these
    possibilities.)  Join sequences with lower estimated cost are considered
    <quote>more fit</> than those with higher cost.  The genetic algorithm
    discards the least fit candidates.  Then new candidates are generated
    by combining genes of more-fit candidates &mdash; that is, by using
    randomly-chosen portions of known low-cost join sequences to create
    new sequences for consideration.  This process is repeated until a
    preset number of join sequences have been considered; then the best
    one found at any time during the search is used to generate the finished
    plan.
   </para>

   <para>
    This process is inherently nondeterministic, because of the randomized
    choices made during both the initial population selection and subsequent
    <quote>mutation</> of the best candidates.  To avoid surprising changes
    of the selected plan, each run of the GEQO algorithm restarts its
    random number generator with the current <xref linkend="guc-geqo-seed">
    parameter setting.  As long as <varname>geqo_seed</> and the other
    GEQO parameters are kept fixed, the same plan will be generated for a
    given query (and other planner inputs such as statistics).  To experiment
    with different search paths, try changing <varname>geqo_seed</>.
   </para>

  </sect2>

  <sect2 id="geqo-future">
   <title>Future Implementation Tasks for
    <productname>PostgreSQL</> <acronym>GEQO</acronym></title>

     <para>
      Work is still needed to improve the genetic algorithm parameter
      settings.
      In file <filename>src/backend/optimizer/geqo/geqo_main.c</filename>,
      routines
      <function>gimme_pool_size</function> and <function>gimme_number_generations</function>,
      we have to find a compromise for the parameter settings
      to satisfy two competing demands:
      <itemizedlist spacing="compact">
       <listitem>
        <para>
         Optimality of the query plan
        </para>
       </listitem>
       <listitem>
        <para>
         Computing time
        </para>
       </listitem>
      </itemizedlist>
     </para>

     <para>
      In the current implementation, the fitness of each candidate join
      sequence is estimated by running the standard planner's join selection
      and cost estimation code from scratch.  To the extent that different
      candidates use similar sub-sequences of joins, a great deal of work
      will be repeated.  This could be made significantly faster by retaining
      cost estimates for sub-joins.  The problem is to avoid expending
      unreasonable amounts of memory on retaining that state.
     </para>

     <para>
      At a more basic level, it is not clear that solving query optimization
      with a GA algorithm designed for TSP is appropriate.  In the TSP case,
      the cost associated with any substring (partial tour) is independent
      of the rest of the tour, but this is certainly not true for query
      optimization.  Thus it is questionable whether edge recombination
      crossover is the most effective mutation procedure.
     </para>

   </sect2>
  </sect1>

 <sect1 id="geqo-biblio">
  <title>Further Reading</title>

  <para>
   The following resources contain additional information about
   genetic algorithms:

   <itemizedlist>
    <listitem>
     <para>
      <ulink url="http://www.aip.de/~ast/EvolCompFAQ/">
      The Hitch-Hiker's Guide to Evolutionary Computation</ulink>, (FAQ for <ulink
      url="news://comp.ai.genetic"></ulink>)
     </para>
    </listitem>

    <listitem>
     <para>
      <ulink url="http://www.red3d.com/cwr/evolve.html">
      Evolutionary Computation and its application to art and design</ulink>, by
      Craig Reynolds
     </para>
    </listitem>

    <listitem>
     <para>
      <xref linkend="ELMA04">
     </para>
    </listitem>

    <listitem>
     <para>
      <xref linkend="FONG">
     </para>
    </listitem>
   </itemizedlist>
  </para>

 </sect1>
</chapter>
@


1.42
log
@Fix URL to "The Hitch-Hiker's Guide to Evolutionary Computation".

Per Andreas Wenk, Andres Freund and Rob Wultsh.  Thanks, Robert Haas, for the
patch.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.41 2009/07/16 20:55:44 tgl Exp $ -->
d28 1
a28 1
  <title id="geqo-title">Genetic Query Optimizer</title>
@


1.41
log
@Make GEQO's planning deterministic by having it start from a predictable
random number seed each time.  This is how it used to work years ago, but
we got rid of the seed reset because it was resetting the main random()
sequence and thus having undesirable effects on the rest of the system.
To fix, establish a private random number state for each execution of
geqo(), and initialize the state using the new GUC variable geqo_seed.
People who want to experiment with different random searches can do so
by changing geqo_seed, but you'll always get the same plan for the same
value of geqo_seed (if holding all other planner inputs constant, of course).

The new state is kept in PlannerInfo by adding a "void *" field reserved
for use by join_search hooks.  Most of the rather bulky code changes in
this commit are just arranging to pass PlannerInfo around to all the GEQO
functions (many of which formerly didn't receive it).

Andres Freund, with some editorialization by Tom
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.40 2007/07/21 04:02:41 tgl Exp $ -->
d331 1
a331 1
      <ulink url="http://www.cs.bham.ac.uk/Mirrors/ftp.de.uu.net/EC/clife/www/location.htm">
@


1.40
log
@Provide a bit more high-level documentation for the GEQO planner.
Per request from Luca Ferrari.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.39 2007/02/16 03:50:29 momjian Exp $ -->
d52 1
a52 1
    B-tree, hash, GiST and GIN in <productname>PostgreSQL</productname>) as 
d91 1
a91 2
    operates through
    nondeterministic, randomized search. The set of possible solutions for the
d118 1
a118 1
    non-random (better than random). 
d262 7
a268 3
    <quote>mutation</> of the best candidates.  Hence different plans may
    be selected from one run to the next, resulting in varying run time
    and varying output row order.
d336 1
a336 1
   
@


1.40.4.1
log
@Fix URL to "The Hitch-Hiker's Guide to Evolutionary Computation".

Per Andreas Wenk, Andres Freund and Rob Wultsh.  Thanks, Robert Haas, for the
patch.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.40 2007-07-21 04:02:41 tgl Exp $ -->
d328 1
a328 1
      <ulink url="http://www.aip.de/~ast/EvolCompFAQ/">
@


1.39
log
@Remove tabs from SGML files to help tag alingment and improve
detection of tabs are added in the future.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.38 2006/09/16 00:30:14 momjian Exp $ -->
a189 5
    Parts of the <acronym>GEQO</acronym> module are adapted from D. Whitley's Genitor
    algorithm.
   </para>

   <para>
d223 5
d234 36
d297 10
@


1.38
log
@Remove emacs info from footer of SGML files.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.37 2006/09/14 11:16:27 teodor Exp $ -->
d53 1
a53 1
	access paths for relations.
@


1.37
log
@GIN documentation and slightly improving GiST docs.
Thanks to  Christopher Kings-Lynne <chris.kingslynne@@gmail.com> for
initial version and Jeff Davis <pgsql@@j-davis.com> for inspection
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.36 2006/03/10 19:10:48 momjian Exp $ -->
a311 17

<!-- Keep this comment at the end of the file
Local variables:
mode:sgml
sgml-omittag:nil
sgml-shorttag:t
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:t
sgml-parent-document:nil
sgml-default-dtd-file:"./reference.ced"
sgml-exposed-tags:nil
sgml-local-catalogs:("/usr/lib/sgml/catalog")
sgml-local-ecat-files:nil
End:
-->
@


1.36
log
@Make $PostgreSQL CVS tags consistent for SGML files.
@
text
@d1 1
a1 1
<!-- $PostgreSQL: pgsql/doc/src/sgml/advanced.sgml,v 1.49 2005/04/09 03:52:43 momjian Exp $ -->
d52 2
a53 2
    B-tree, hash, GiST in <productname>PostgreSQL</productname>) as access
    paths for relations.
@


1.35
log
@Minor improvements to GEQO documentation.
@
text
@d1 1
a1 4
<!--
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.34 2005/11/07 17:36:44 tgl Exp $
Genetic Optimizer
-->
@


1.34
log
@R-tree is dead ... long live GiST.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.33 2005/10/25 13:38:09 momjian Exp $
d49 2
a50 2
    alternative plans to answer a query grows exponentially with the
    number of joins included in it. Further optimization effort is
d60 6
a65 7
    The current <productname>PostgreSQL</productname> optimizer
    implementation performs a <firstterm>near-exhaustive
    search</firstterm> over the space of alternative strategies. This
    algorithm, first introduced in the <quote>System R</quote>
    database, produces a near-optimal join order, but can take an
    enormous amount of time and memory space when the number of joins
    in the query grows large. This makes the ordinary
d72 7
a78 10
    Technology, in Freiberg, Germany, encountered the described problems as its
    folks wanted to take the <productname>PostgreSQL</productname> DBMS as the backend for a decision
    support knowledge based system for the maintenance of an electrical
    power grid. The DBMS needed to handle large join queries for the
    inference machine of the knowledge based system.
   </para>

   <para>
    Performance difficulties in exploring the space of possible query
    plans created the demand for a new optimization technique to be developed.
d83 1
a83 1
    <firstterm>Genetic Algorithm</firstterm> to solve the join
@


1.33
log
@Update the URL for "The Hitch-Hiker's Guide to Evolutionary Computation".

Sergey E. Koposov
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.32 2005/04/12 03:16:50 tgl Exp $
d54 2
a55 2
    and a diversity of <firstterm>indexes</firstterm> (e.g., R-tree,
    B-tree, hash in <productname>PostgreSQL</productname>) as access
@


1.32
log
@Fix broken markup.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.31 2005/04/09 03:52:43 momjian Exp $
d289 1
a289 1
      <ulink url="http://surf.de.uu.net/encore/www/">
@


1.31
log
@We don't put URL's in ulink's because the URL is always generated, but
we can put words in ulink and the URL will still be printed.

per Peter
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.30 2005/03/31 03:54:38 momjian Exp $
d290 1
a290 1
      The Hitch-Hiker's Guide to Evolutionary Computation/ulink>, (FAQ for <ulink
@


1.30
log
@Adjust SGML ulink tags to the URL is always displayed in our printed docs.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.29 2005/02/21 02:21:00 neilc Exp $
d289 2
a290 2
      The Hitch-Hiker's Guide to Evolutionary Computation,
      <ulink url="http://surf.de.uu.net/encore/www/"></ulink>, (FAQ for <ulink
d297 2
a298 2
      Evolutionary Computation and its application to art and design,
      <ulink url="http://www.red3d.com/cwr/evolve.html"></ulink>, by
@


1.29
log
@Updates to the bibliography. Patch from Michael Fuhr.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.28 2005/01/22 22:56:36 momjian Exp $
d289 3
a291 3
      <ulink url="http://surf.de.uu.net/encore/www/">The Hitch-Hiker's
      Guide to Evolutionary Computation</ulink> (FAQ for <ulink
      url="news://comp.ai.genetic">comp.ai.genetic</ulink>)
d297 2
a298 2
      <ulink url="http://www.red3d.com/cwr/evolve.html">Evolutionary
       Computation and its application to art and design</ulink> by
@


1.28
log
@More < and > cleanups converted to ampersands.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.27 2005/01/05 23:42:03 tgl Exp $
d305 1
a305 1
      <xref linkend="ELMA99">
@


1.27
log
@More minor updates and copy-editing.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.26 2003/11/29 19:51:37 pgsql Exp $
d150 1
a150 1
|>>>>>>>>>>>  Algorithm GA  <<<<<<<<<<<<<<|
@


1.27.4.1
log
@More < and > cleanups converted to ampersands.
@
text
@d2 1
a2 1
$PostgreSQL: pgsql/doc/src/sgml/geqo.sgml,v 1.27 2005/01/05 23:42:03 tgl Exp $
d150 1
a150 1
|&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;&gt;  Algorithm GA  &lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;|
@


1.26
log
@
$Header: -> $PostgreSQL Changes ...
@
text
@d2 1
a2 1
$PostgreSQL: /cvsroot/pgsql-server/doc/src/sgml/geqo.sgml,v 1.25 2003/11/24 19:08:02 petere Exp $
d68 1
a68 2
    inappropriate for database application domains that involve the
    need for extensive queries, such as artificial intelligence.
d99 1
a99 1
    determined, randomized search. The set of possible solutions for the
d178 3
a180 2
    The <acronym>GEQO</acronym> module is intended for the solution of the query
    optimization problem similar to a traveling salesman problem (<acronym>TSP</acronym>).
d183 1
a183 1
    E. g., the query tree
d248 2
a249 1
      In file <filename>backend/optimizer/geqo/geqo_params.c</filename>, routines
d255 3
a257 3
	<para>
	 Optimality of the query plan
	</para>
d260 3
a262 3
	<para>
	 Computing time
	</para>
d267 9
d280 1
a280 1
  <title>Further Readings</title>
@


1.25
log
@Upgrade to DocBook V4.2 SGML.
@
text
@d2 1
a2 1
$Header: /cvsroot/pgsql-server/doc/src/sgml/geqo.sgml,v 1.24 2003/09/29 18:18:35 momjian Exp $
@


1.24
log
@This patch makes a few incremental improvements to geqo.sgml and
arch-dev.sgml

Neil Conway
@
text
@d2 1
a2 1
$Header: /cvsroot/pgsql-server/doc/src/sgml/geqo.sgml,v 1.23 2002/01/20 22:19:56 petere Exp $
d7 1
a7 1
  <docinfo>
d29 1
a29 1
  </docinfo>
@


1.23
log
@Spell-check and markup police
@
text
@d2 1
a2 1
$Header: /cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.22 2002/01/08 18:03:26 petere Exp $
d31 1
a31 1
  <title>Genetic Query Optimization</title>
d47 10
a56 9
    Among all relational operators the most difficult one to process and
    optimize is the <firstterm>join</firstterm>. The number of alternative plans to answer a query
    grows exponentially with the number of joins included in it. Further
    optimization effort is caused by the support of a variety of
    <firstterm>join methods</firstterm>
    (e.g., nested loop, hash join, merge join in <productname>PostgreSQL</productname>) to
    process individual joins and a diversity of
    <firstterm>indexes</firstterm> (e.g., R-tree,
    B-tree, hash in <productname>PostgreSQL</productname>) as access paths for relations.
d61 9
a69 5
    implementation performs a <firstterm>near-exhaustive search</firstterm>
    over the space of alternative strategies. This query 
    optimization technique is inadequate to support database application
    domains that involve the need for extensive queries, such as artificial
    intelligence.
d83 1
a83 1
    plans created the demand for a new optimization technique being developed.
d87 4
a90 2
    In the following we propose the implementation of a <firstterm>Genetic Algorithm</firstterm>
    as an option for the database query optimization problem.
d218 4
a221 4
       Usage of <firstterm>edge recombination crossover</firstterm> which is
       especially suited
       to keep edge losses low for the solution of the
       <acronym>TSP</acronym> by means of a <acronym>GA</acronym>;
@


1.22
log
@Repair the GEQO bibliography.  It was marked up all wrong.  Move some of
the info to the main bibliography.
@
text
@d2 1
a2 1
$Header: /cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.21 2002/01/08 05:39:26 momjian Exp $
d96 1
a96 1
    The degree of adaption of an individual to its environment is specified
d150 1
a150 1
| evalute FITNESS of P(t)                 |
d160 1
a160 1
|   | evalute FITNESS of P''(t)           |
d185 1
a185 1
    then '2', where 1, 2, 3, 4 are relids within the
@


1.21
log
@Update bad URL's with good ones.
@
text
@d2 1
a2 1
$Header: /cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.20 2001/11/21 05:53:41 thomas Exp $
d262 23
a284 7
   <bibliography id="geqo-biblio">
    <title>
     References
    </title>
    <para>Reference information for <acronym>GEQ</acronym> algorithms.
    </para>
    <biblioentry>
d286 5
a290 94
     <bookbiblio>
      <title>
       The Hitch-Hiker's Guide to Evolutionary Computation
      </title>
      <authorgroup>
       <author>
	<firstname>J&ouml;rg</firstname>
	<surname>Heitk&ouml;tter</surname>
       </author>
       <author>
	<firstname>David</firstname>
	<surname>Beasley</surname>
       </author>
      </authorgroup>
      <publisher>
       <publishername>
	InterNet resource
       </publishername>
      </publisher>
      <abstract>
       <para>
	FAQ in <ulink url="news://comp.ai.genetic">comp.ai.genetic</ulink>
	is available at <ulink
	url="http://surf.de.uu.net/encore/www/">here</ulink>.
       </para>
      </abstract>
     </bookbiblio>

     <bookbiblio>
      <title>
       Evolutionary Computation and its application to art and design

      </title>
      <authorgroup>
       <author>
	<firstname>Craig</firstname>
	<surname>Reynolds</surname>
       </author>
      </authorgroup>
      <publisher>
       <publishername>
	InterNet resource
       </publishername>
      </publisher>
      <abstract>
       <para>
	The URL is <ulink
	url="http://www.red3d.com/cwr/evolve.html">here</ulink>.
       </para>
      </abstract>
     </bookbiblio>

     <bookbiblio>
      <title>
       The Design and Implementation of the Postgres Query Optimizer
      </title>
      <authorgroup>
       <author>
	<firstname>Z.</firstname>
	<surname>Fong</surname>
       </author>
      </authorgroup>
      <publisher>
       <publishername>
	University of California, Berkeley Computer Science Department
       </publishername>
      </publisher>
      <abstract>
       <para>
	File <filename>planner/Report.ps</filename> in the 'postgres-papers' distribution.
       </para>
      </abstract>
     </bookbiblio>

     <bookbiblio>
      <title>
       Fundamentals of Database Systems
      </title>
      <authorgroup>
       <author>
	<firstname>R.</firstname>
	<surname>Elmasri</surname>
       </author>
       <author>
	<firstname>S.</firstname>
	<surname>Navathe</surname>
       </author>
      </authorgroup>
      <publisher>
       <publishername>
	The Benjamin/Cummings Pub., Inc.
       </publishername>
      </publisher>
     </bookbiblio>
d292 7
a298 2
    </biblioentry>
   </bibliography>
d300 2
a301 1
 </chapter>
@


1.20
log
@Deprecate 'current' for date/time input.
Fix up references to "PostgreSQL" rather than "Postgres". Was roughly
 evenly split between the two before. ref/ files not yet done.
@
text
@d2 1
a2 1
$Header: /cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.19 2001/10/09 18:46:00 petere Exp $
d293 25
a317 1
	 url="ftp://ftp.Germany.EU.net/pub/research/softcomp/EC/Welcome.html">Encore</ulink>.
@


1.19
log
@Bunch of copy fitting and style sheet tweakage to get decent looking print
output (from pdfjadetex).  Also updated instructions to install documentation
processing toolchain.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.18 2001/09/13 15:55:22 petere Exp $
d52 1
a52 1
    (e.g., nested loop, hash join, merge join in <productname>Postgres</productname>) to
d55 1
a55 1
    B-tree, hash in <productname>Postgres</productname>) as access paths for relations.
d59 1
a59 1
    The current <productname>Postgres</productname> optimizer
d70 1
a70 1
    folks wanted to take the <productname>Postgres</productname> DBMS as the backend for a decision
d169 1
a169 1
   <title>Genetic Query Optimization (<acronym>GEQO</acronym>) in Postgres</title>
d186 1
a186 1
    <productname>Postgres</productname> optimizer.
d196 1
a196 1
    implementation in <productname>Postgres</productname>
d229 1
a229 1
    the <productname>Postgres</productname> query optimizer to
@


1.18
log
@Replace ASCII-quotes with proper markup.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.17 2001/05/17 21:50:15 petere Exp $
d49 1
a49 1
    grows exponentially with the number of <command>join</command>s included in it. Further
d53 1
a53 1
    process individual <command>join</command>s and a diversity of
d72 1
a72 1
    power grid. The DBMS needed to handle large <command>join</command> queries for the
d88 1
a88 1
   <title>Genetic Algorithms (<acronym>GA</acronym>)</title>
d91 1
a91 1
    The <acronym>GA</acronym> is a heuristic optimization method which
d121 1
d123 2
a124 3
    <programlisting>
Structured Diagram of a <acronym>GA</acronym>:
---------------------------
d126 15
a140 2
P(t)    generation of ancestors at a time t
P''(t)  generation of descendants at a time t
d142 1
d164 2
a165 2
    </programlisting>
   </para>
d175 1
a175 1
    represents the <command>join</command> order from one relation of the query to the next.
d177 1
a177 1
    <programlisting>
d182 1
a182 1
    </programlisting>
d230 1
a230 1
    support large <command>join</command> queries effectively through
d260 1
a343 1
  </sect1>
@


1.17
log
@Change "indices" to "indexes", per OED.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.16 2001/04/20 15:52:33 thomas Exp $
d117 1
a117 1
    According to the "comp.ai.genetic" <acronym>FAQ</acronym> it cannot be stressed too
@


1.16
log
@Final markup for 7.1 hardcopy release. Mostly typo and table-fitting
 adjustments. Note that many tables are being abused with *really* long
 description columns. Should probably shrink those columns to be more
 concise, and move some of the info to follow-on reference notes.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.15 2000/12/22 21:51:57 petere Exp $
d54 2
a55 2
    <firstterm>indices</firstterm> (e.g., r-tree,
    b-tree, hash in <productname>Postgres</productname>) as access paths for relations.
@


1.15
log
@Make use of <email> tag for marking up email addresses.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.14 2000/12/16 22:44:47 tgl Exp $
d31 1
a31 1
  <title>Genetic Query Optimization in Database Systems</title>
@


1.14
log
@Update some obsolete info about GEQO.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.13 2000/09/29 20:21:33 petere Exp $
d37 1
a37 1
     Written by <ulink url="mailto:utesch@@aut.tu-freiberg.de">Martin Utesch</ulink>
@


1.13
log
@Add `id' attributes to all `chapter' and `sect1' tags, to generate useful
names for the HTML files (e.g., not x4856.htm).
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.12 2000/09/18 20:11:37 petere Exp $
d52 1
a52 1
    (e.g., nested loop, index scan, merge join in <productname>Postgres</productname>) to
d60 2
a61 2
    implementation performs a <firstterm>near-
     exhaustive search</firstterm> over the space of alternative strategies. This query
d77 2
a78 2
    Performance difficulties within exploring the space of possible query
    plans arose the demand for a new optimization technique being developed.
d91 2
a92 1
    The <acronym>GA</acronym> is a heuristic optimization method which operates through 
d95 1
a95 1
    <firstterm>erm>popula</firstterm>erm> of <firstterm>individuals</firstterm>.
d171 2
a172 1
    then '2', where 1, 2, 3, 4 are relids in <productname>Postgres</productname>.
d197 2
a198 1
       Usage of <firstterm>edge recombination crossover</firstterm> which is especially suited
d200 1
a200 1
       <acronym>cro</acronym>cronym> by means of a <acronym>GA</acronym>;
d214 4
a217 19
    The <acronym>GEQO</acronym> module gives the following benefits to
    the <productname>Postgres</productname> DBMS
    compared to the <productname>Postgres</productname> query optimizer implementation:

    <itemizedlist spacing="compact" mark="bullet">
     <listitem>
      <para>
       Handling of large <command>join</command> queries through non-exhaustive search;
      </para>
     </listitem>

     <listitem>
      <para>
       Improved cost size approximation of query plans since no longer
       plan merging is needed (the <acronym>GEQO</acronym> module evaluates the cost for a
       query plan as an individual).
      </para>
     </listitem>
    </itemizedlist>
d220 1
a220 3
  </sect1>

  <sect1 id="geqo-future">
a223 6
   <sect2>
    <title>Basic Improvements</title>

    <sect3>
     <title>Improve genetic algorithm parameter settings</title>

d225 2
a243 17
    </sect3>

    <sect3>
     <title>Find better solution for integer overflow</title>

     <para>
      In file <filename>backend/optimizer/geqo/geqo_eval.c</filename>, routine
      <function>geqo_joinrel_size</function>,
      the present hack for MAXINT overflow is to set the <productname>Postgres</productname> integer
      value of <structfield>rel->size</structfield> to its logarithm.
      Modifications of <structname>Rel</structname> in <filename>backend/nodes/relation.h</filename> will
      surely have severe impacts on the whole <productname>Postgres</productname> implementation.
     </para>
    </sect3>

    <sect3>
     <title>Find solution for exhausted memory</title>
a244 11
     <para>
      Memory exhaustion may occur with more than 10 relations involved in a query.
      In file <filename>backend/optimizer/geqo/geqo_eval.c</filename>, routine
      <function>gimme_tree</function> is recursively called.
      Maybe I forgot something to be freed correctly, but I dunno what.
      Of course the <structname>rel</structname> data structure of the
      <command>join</command> keeps growing and
      growing the more relations are packed into it.
      Suggestions are welcome :-(
     </para>
    </sect3>
a245 1

@


1.12
log
@Move contrib/odbc/odbc.sql together with odbc driver, remove old files;
revise ODBC "installation instructions"; update some other outdated
documentation; update contrib documentation
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.11 2000/08/23 05:59:02 thomas Exp $
d43 1
a43 1
  <sect1>
d87 1
a87 1
  <sect1>
d153 1
a153 1
  <sect1>
d234 1
a234 1
  <sect1>
@


1.11
log
@Fix several <ulink> tags which refer to e-mail addresses
 but were missing the "mailto:" prefix.
Fix typo.
Thanks to Neil Conway <nconway@@klamath.dyndns.org> for the heads-up.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.10 2000/06/28 03:30:53 tgl Exp $
d236 1
a236 1
    <productname>ame>Post</productname>ame> <acronym>GEQO</acronym></title>
@


1.10
log
@First phase of memory management rewrite (see backend/utils/mmgr/README
for details).  It doesn't really do that much yet, since there are no
short-term memory contexts in the executor, but the infrastructure is
in place and long-term contexts are handled reasonably.  A few long-
standing bugs have been fixed, such as 'VACUUM; anything' in a single
query string crashing.  Also, out-of-memory is now considered a
recoverable ERROR, not FATAL.
Eliminate a large amount of crufty, now-dead code in and around
memory management.
Fix problem with holding off SIGTRAP, SIGSEGV, etc in postmaster and
backend startup.
@
text
@d2 1
a2 1
$Header: /home/projects/pgsql/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.9 2000/03/31 03:27:40 thomas Exp $
d6 114
a119 108
<Chapter Id="geqo">
<DocInfo>
<Author>
<FirstName>Martin</FirstName>
<SurName>Utesch</SurName>
<Affiliation>
<Orgname>
University of Mining and Technology
</Orgname>
<Orgdiv>
Institute of Automatic Control
</Orgdiv>
<Address>
<City>
Freiberg
</City>
<Country>
Germany
</Country>
</Address>
</Affiliation>
</Author>
<Date>1997-10-02</Date>
</DocInfo>

<Title>Genetic Query Optimization in Database Systems</Title>

<Para>
<Note>
<Title>Author</Title>
<Para>
Written by <ULink url="utesch@@aut.tu-freiberg.de">Martin Utesch</ULink>
for the Institute of Automatic Control at the University of Mining and Technology in Freiberg, Germany.
</Para>
</Note>
</para>

<Sect1>
<Title>Query Handling as a Complex Optimization Problem</Title>

<Para>
   Among all relational operators the most difficult one to process and
optimize is the <FirstTerm>join</FirstTerm>. The number of alternative plans to answer a query
grows exponentially with the number of <Command>join</Command>s included in it. Further
optimization effort is caused by the support of a variety of <FirstTerm>join methods</FirstTerm>
 (e.g., nested loop, index scan, merge join in <ProductName>Postgres</ProductName>) to
process individual <Command>join</Command>s and a diversity of <FirstTerm>indices</FirstTerm> (e.g., r-tree,
b-tree, hash in <ProductName>Postgres</ProductName>) as access paths for relations.
</para>

<Para>
   The current <ProductName>Postgres</ProductName> optimizer implementation performs a <FirstTerm>near-
exhaustive search</FirstTerm> over the space of alternative strategies. This query
optimization technique is inadequate to support database application
domains that involve the need for extensive queries, such as artificial
intelligence.
</para>

<Para>
   The Institute of Automatic Control at the University of Mining and
Technology, in Freiberg, Germany, encountered the described problems as its
folks wanted to take the <ProductName>Postgres</ProductName> DBMS as the backend for a decision
support knowledge based system for the maintenance of an electrical
power grid. The DBMS needed to handle large <Command>join</Command> queries for the
inference machine of the knowledge based system.
</para>

<Para>
   Performance difficulties within exploring the space of possible query
plans arose the demand for a new optimization technique being developed.
</para>

<Para>
   In the following we propose the implementation of a <FirstTerm>Genetic Algorithm</FirstTerm>
 as an option for the database query optimization problem.
</para>
</sect1>

<Sect1>
<Title>Genetic Algorithms (<Acronym>GA</Acronym>)</Title>

<Para>
   The <Acronym>GA</Acronym> is a heuristic optimization method which operates through 
determined, randomized search. The set of possible solutions for the
optimization problem is considered as a <FirstTerm>population</FirstTerm> of <FirstTerm>individuals</FirstTerm>.
The degree of adaption of an individual to its environment is specified
by its <FirstTerm>fitness</FirstTerm>.
</para>

<Para>
   The coordinates of an individual in the search space are represented
by <FirstTerm>chromosomes</FirstTerm>, in essence a set of character strings. A <FirstTerm>gene</FirstTerm> is a
subsection of a chromosome which encodes the value of a single parameter
being optimized. Typical encodings for a gene could be <FirstTerm>binary</FirstTerm> or
<FirstTerm>integer</FirstTerm>.
</para>

<Para>
   Through simulation of the evolutionary operations <FirstTerm>recombination</FirstTerm>,
<FirstTerm>mutation</FirstTerm>, and <FirstTerm>selection</FirstTerm> new generations of search points are found
that show a higher average fitness than their ancestors.
</para>

<Para>
   According to the "comp.ai.genetic" <Acronym>FAQ</Acronym> it cannot be stressed too
strongly that a <Acronym>GA</Acronym> is not a pure random search for a solution to a
problem. A <Acronym>GA</Acronym> uses stochastic processes, but the result is distinctly
non-random (better than random). 
d121 2
a122 2
<ProgramListing>
Structured Diagram of a <Acronym>GA</Acronym>:
d149 223
a371 217
</ProgramListing>
</para>
</sect1>

<Sect1>
<Title>Genetic Query Optimization (<Acronym>GEQO</Acronym>) in Postgres</Title>

<Para>
   The <Acronym>GEQO</Acronym> module is intended for the solution of the query
optimization problem similar to a traveling salesman problem (<Acronym>TSP</Acronym>).
Possible query plans are encoded as integer strings. Each string
represents the <Command>join</Command> order from one relation of the query to the next.
E. g., the query tree
<ProgramListing>
       /\
      /\ 2
     /\ 3
    4  1
</ProgramListing>
is encoded by the integer string '4-1-3-2',
which means, first join relation '4' and '1', then '3', and
then '2', where 1, 2, 3, 4 are relids in <ProductName>Postgres</ProductName>.
</para>

<Para>
   Parts of the <Acronym>GEQO</Acronym> module are adapted from D. Whitley's Genitor
algorithm.
</para>

<Para>
   Specific characteristics of the <Acronym>GEQO</Acronym> implementation in <ProductName>Postgres</ProductName>
are:

<ItemizedList Mark="bullet" Spacing="compact">
<ListItem>
<Para>
Usage of a <FirstTerm>steady state</FirstTerm> <Acronym>GA</Acronym> (replacement of the least fit
   individuals in a population, not whole-generational replacement)
   allows fast convergence towards improved query plans. This is
   essential for query handling with reasonable time;
</Para>
</ListItem>

<ListItem>
<Para>
Usage of <FirstTerm>edge recombination crossover</FirstTerm> which is especially suited
   to keep edge losses low for the solution of the <Acronym>TSP</Acronym> by means of a <Acronym>GA</Acronym>;
</Para>
</ListItem>

<ListItem>
<Para>
Mutation as genetic operator is deprecated so that no repair
   mechanisms are needed to generate legal <Acronym>TSP</Acronym> tours.
</Para>
</ListItem>
</ItemizedList>
</para>

<Para>
   The <Acronym>GEQO</Acronym> module gives the following benefits to the <ProductName>Postgres</ProductName> DBMS
compared to the <ProductName>Postgres</ProductName> query optimizer implementation:

<ItemizedList Mark="bullet" Spacing="compact">
<ListItem>
<Para>
Handling of large <Command>join</Command> queries through non-exhaustive search;
</Para>
</ListItem>

<ListItem>
<Para>
Improved cost size approximation of query plans since no longer
   plan merging is needed (the <Acronym>GEQO</Acronym> module evaluates the cost for a
   query plan as an individual).
</Para>
</ListItem>
</ItemizedList>
</para>

</Sect1>

<Sect1>
<Title>Future Implementation Tasks for <ProductName>Postgres</ProductName> <Acronym>GEQO</Acronym></Title>

<Sect2>
<Title>Basic Improvements</Title>

<Sect3>
<Title>Improve genetic algorithm parameter settings</Title>

<Para>
In file <FileName>backend/optimizer/geqo/geqo_params.c</FileName>, routines
<Function>gimme_pool_size</Function> and <Function>gimme_number_generations</Function>,
we have to find a compromise for the parameter settings
to satisfy two competing demands:
<ItemizedList Spacing="compact">
<ListItem>
<Para>
Optimality of the query plan
</Para>
</ListItem>
<ListItem>
<Para>
Computing time
</Para>
</ListItem>
</ItemizedList>
</para>
</sect3>

<Sect3>
<Title>Find better solution for integer overflow</Title>

<Para>
In file <FileName>backend/optimizer/geqo/geqo_eval.c</FileName>, routine
<Function>geqo_joinrel_size</Function>,
the present hack for MAXINT overflow is to set the <ProductName>Postgres</ProductName> integer
value of <StructField>rel->size</StructField> to its logarithm.
Modifications of <StructName>Rel</StructName> in <FileName>backend/nodes/relation.h</FileName> will
surely have severe impacts on the whole <ProductName>Postgres</ProductName> implementation.
</para>
</sect3>

<Sect3>
<Title>Find solution for exhausted memory</Title>

<Para>
Memory exhaustion may occur with more than 10 relations involved in a query.
In file <FileName>backend/optimizer/geqo/geqo_eval.c</FileName>, routine
<Function>gimme_tree</Function> is recursively called.
Maybe I forgot something to be freed correctly, but I dunno what.
Of course the <StructName>rel</StructName> data structure of the <Command>join</Command> keeps growing and
growing the more relations are packed into it.
Suggestions are welcome :-(
</para>
</sect3>
</sect2>


<BIBLIOGRAPHY Id="geqo-biblio">
<TITLE>
References
</TITLE>
<PARA>Reference information for <Acronym>GEQ</Acronym> algorithms.
</PARA>
<BIBLIOENTRY>

<BOOKBIBLIO>
<TITLE>
The Hitch-Hiker's Guide to Evolutionary Computation
</TITLE>
<AUTHORGROUP>
<AUTHOR>
<FIRSTNAME>J&ouml;rg</FIRSTNAME>
<SURNAME>Heitk&ouml;tter</SURNAME>
</AUTHOR>
<AUTHOR>
<FIRSTNAME>David</FIRSTNAME>
<SURNAME>Beasley</SURNAME>
</AUTHOR>
</AUTHORGROUP>
<PUBLISHER>
<PUBLISHERNAME>
InterNet resource
</PUBLISHERNAME>
</PUBLISHER>
<ABSTRACT>
<Para>
FAQ in <ULink url="news://comp.ai.genetic">comp.ai.genetic</ULink>
is available at <ULink url="ftp://ftp.Germany.EU.net/pub/research/softcomp/EC/Welcome.html">Encore</ULink>.
</Para>
</ABSTRACT>
</BOOKBIBLIO>

<BOOKBIBLIO>
<TITLE>
The Design and Implementation of the Postgres Query Optimizer
</TITLE>
<AUTHORGROUP>
<AUTHOR>
<FIRSTNAME>Z.</FIRSTNAME>
<SURNAME>Fong</SURNAME>
</AUTHOR>
</AUTHORGROUP>
<PUBLISHER>
<PUBLISHERNAME>
University of California, Berkeley Computer Science Department
</PUBLISHERNAME>
</PUBLISHER>
<ABSTRACT>
<Para>
File <FileName>planner/Report.ps</FileName> in the 'postgres-papers' distribution.
</Para>
</ABSTRACT>
</BOOKBIBLIO>

<BOOKBIBLIO>
<TITLE>
Fundamentals of Database Systems
</TITLE>
<AUTHORGROUP>
<AUTHOR>
<FIRSTNAME>R.</FIRSTNAME>
<SURNAME>Elmasri</SURNAME>
</AUTHOR>
<AUTHOR>
<FIRSTNAME>S.</FIRSTNAME>
<SURNAME>Navathe</SURNAME>
</AUTHOR>
</AUTHORGROUP>
<PUBLISHER>
<PUBLISHERNAME>
The Benjamin/Cummings Pub., Inc.
</PUBLISHERNAME>
</PUBLISHER>
</BOOKBIBLIO>
d373 2
a374 2
</BIBLIOENTRY>
</BIBLIOGRAPHY>
d376 2
a377 2
</sect1>
</Chapter>
@


1.9
log
@Minor markup changes. Refer to (".../catalog") in the emacs hints.
@
text
@d2 1
a2 1
$Header: /usr/local/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.8 1999/03/30 15:25:56 thomas Exp $
a229 16

<Sect3>
<Title>Improve freeing of memory when query is already processed</Title>

<Para>
With large <Command>join</Command> queries the computing time spent for the genetic query
optimization seems to be a mere <Emphasis>fraction</Emphasis> of the time
 <ProductName>Postgres</ProductName>
needs for freeing memory via routine <Function>MemoryContextFree</Function>,
file <FileName>backend/utils/mmgr/mcxt.c</FileName>.
Debugging showed that it get stucked in a loop of routine
<Function>OrderedElemPop</Function>, file <FileName>backend/utils/mmgr/oset.c</FileName>.
The same problems arise with long queries when using the normal
<ProductName>Postgres</ProductName> query optimization algorithm.
</para>
</sect3>
@


1.8
log
@Fix up small markup problems. Force omit-tags to nil so we have tag
 completion as required by the newest DocBook conventions.
@
text
@d2 1
a2 1
$Header: /opt/postgres/cvs/pgsql/doc/src/sgml/geqo.sgml,v 1.7 1999/02/19 01:57:08 thomas Exp $
a3 16

$Log: geqo.sgml,v $
Revision 1.7  1999/02/19 01:57:08  thomas
Fix SGML markup from last content changes.

Revision 1.6  1999/02/18 05:26:17  momjian
Enable bushy plans by default.

Revision 1.5  1998/12/29 02:24:15  thomas
Clean up to ensure tag completion as required by the newest versions
 of Norm's Modular Style Sheets and jade/docbook.
From Vince Vielhaber <vev@@michvhf.com>.

Revision 1.4  1998/08/15 06:55:05  thomas
Change Id field in chapter tag to change html output file name.

d385 1
a385 1
mode: sgml
d395 1
a395 1
sgml-local-catalogs:"/usr/lib/sgml/catalog"
@


1.7
log
@Fix SGML markup from last content changes.
@
text
@d2 1
a2 1
$Header: /opt/postgres/cvs/pgsql/doc/src/sgml/geqo.sgml,v 1.6 1999/02/18 05:26:17 momjian Exp $
d6 3
d398 17
@


1.6
log
@Enable bushy plans by default.
@
text
@d2 1
a2 1
$Header: /usr/local/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.5 1998/12/29 02:24:15 thomas Exp $
d6 3
a310 2
<Sect2>
<Title>Further Improvements</Title>
a392 1
</sect2>
@


1.5
log
@Clean up to ensure tag completion as required by the newest versions
 of Norm's Modular Style Sheets and jade/docbook.
From Vince Vielhaber <vev@@michvhf.com>.
@
text
@d2 1
a2 1
$Header: /usr/local/cvsroot/pgsql/doc/src/sgml/geqo.sgml,v 1.4 1998/08/15 06:55:05 thomas Exp $
d6 5
a309 5

<Para>
Enable bushy query tree processing within <ProductName>Postgres</ProductName>;
that may improve the quality of query plans.
</para>
@


1.4
log
@Change Id field in chapter tag to change html output file name.
@
text
@d2 1
a2 1
$Header$
d5 4
a8 1
$Log$
d46 1
d59 1
d67 1
d76 1
d81 1
d86 2
a87 1

d98 1
d106 1
d112 1
d149 2
d170 1
d175 1
d205 1
d226 1
d249 2
d272 2
d285 2
d299 3
a301 1

d309 1
d392 2
@


1.3
log
@Add port information for NS32532.
Fix Id tag in chapter headings to allow meaningful file names in html.
@
text
@d1 7
d284 1
a284 1
<BIBLIOGRAPHY Id="geqo-references">
@


1.2
log
@Convert body of chapter to SGML. Was embedded text from original doc.
@
text
@d1 1
a1 1
<Chapter>
d277 1
a277 1
<BIBLIOGRAPHY>
@


1.1
log
@SGML source for new documentation.
@
text
@d6 16
d23 1
d29 7
a35 2
<ProgramListing>
<ULink url="utesch@@aut.tu-freiberg.de">Martin Utesch</ULink>
d37 2
a38 9
          Institute of Automatic Control
        University of Mining and Technology
                 Freiberg, Germany

                    02/10/1997


1.) Query Handling as a Complex Optimization Problem
====================================================
d40 1
d42 6
a47 6
optimize is the JOIN. The number of alternative plans to answer a query
grows exponentially with the number of JOINs included in it. Further
optimization effort is caused by the support of a variety of *JOIN
methods* (e.g., nested loop, index scan, merge join in Postgres) to
process individual JOINs and a diversity of *indices* (e.g., r-tree,
b-tree, hash in Postgres) as access paths for relations.
d49 3
a51 2
   The current Postgres optimizer implementation performs a *near-
exhaustive search* over the space of alternative strategies. This query
d56 1
d59 1
a59 1
folks wanted to take the Postgres DBMS as the backend for a decision
d61 1
a61 1
power grid. The DBMS needed to handle large JOIN queries for the
d64 1
d68 3
a70 2
   In the following we propose the implementation of a *Genetic
Algorithm* as an option for the database query optimization problem.
d73 2
a74 2
2.) Genetic Algorithms (GA)
===========================
d76 2
a77 1
   The GA is a heuristic optimization method which operates through 
d79 1
a79 1
optimization problem is considered as a *population* of *individuals*.
d81 1
a81 1
by its *fitness*.
d83 1
d85 1
a85 1
by *chromosomes*, in essence a set of character strings. A *gene* is a
d87 2
a88 2
being optimized. Typical encodings for a gene could be *binary* or
*integer*.
d90 3
a92 2
   Through simulation of the evolutionary operations *recombination*,
*mutation*, and *selection* new generations of search points are found
d95 4
a98 3
   According to the "comp.ai.genetic" FAQ it cannot be stressed too
strongly that a GA is not a pure random search for a solution to a
problem. A GA uses stochastic processes, but the result is distinctly
d101 2
a102 1
Structured Diagram of a GA:
d129 1
d131 2
d134 3
a136 5
3.) Genetic Query Optimization (GEQO) in PostgreSQL
===================================================

   The GEQO module is intended for the solution of the query
optimization problem similar to a traveling salesman problem (TSP).
d138 9
a146 5
represents the JOIN order from one relation of the query to the next.
E. g., the query tree  /\
                      /\ 2
                     /\ 3
                    4  1  is encoded by the integer string '4-1-3-2',
d148 1
a148 1
then '2', where 1, 2, 3, 4 are relids in PostgreSQL.
d150 2
a151 1
   Parts of the GEQO module are adapted from D. Whitley's Genitor
d154 2
a155 1
   Specific characteristics of the GEQO implementation in PostgreSQL
d158 4
a161 1
o  usage of a *steady state* GA (replacement of the least fit
d165 2
d168 6
a173 2
o  usage of *edge recombination crossover* which is especially suited
   to keep edge losses low for the solution of the TSP by means of a GA;
d175 7
a181 2
o  mutation as genetic operator is deprecated so that no repair
   mechanisms are needed to generate legal TSP tours.
d183 3
a185 2
   The GEQO module gives the following benefits to the PostgreSQL DBMS
compared to the Postgres query optimizer implementation:
d187 6
a192 1
o  handling of large JOIN queries through non-exhaustive search;
d194 4
a197 2
o  improved cost size approximation of query plans since no longer
   plan merging is needed (the GEQO module evaluates the cost for a
d199 8
d208 2
d211 16
a226 2
References
==========
d228 3
a230 46
J. Heitk"otter, D. Beasley:
---------------------------
   "The Hitch-Hicker's Guide to Evolutionary Computation",
   FAQ in 'comp.ai.genetic',
   'ftp://ftp.Germany.EU.net/pub/research/softcomp/EC/Welcome.html'

Z. Fong:
--------
   "The Design and Implementation of the Postgres Query Optimizer",
   file 'planner/Report.ps' in the 'postgres-papers' distribution

R. Elmasri, S. Navathe:
-----------------------
   "Fundamentals of Database Systems",
   The Benjamin/Cummings Pub., Inc.


=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=
*         Things left to done for the PostgreSQL                    *
=           Genetic Query Optimization (GEQO)                       =
*              module implementation                                *
=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=
* Martin Utesch		      * Institute of Automatic Control      *
=                             = University of Mining and Technology =
* utesch@@aut.tu-freiberg.de   * Freiberg, Germany                   *
=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=


1.) Basic Improvements
===============================================================

a) improve freeing of memory when query is already processed:
-------------------------------------------------------------
with large JOIN queries the computing time spent for the genetic query
optimization seems to be a mere *fraction* of the time Postgres
needs for freeing memory via routine 'MemoryContextFree',
file 'backend/utils/mmgr/mcxt.c';
debugging showed that it get stucked in a loop of routine
'OrderedElemPop', file 'backend/utils/mmgr/oset.c';
the same problems arise with long queries when using the normal
Postgres query optimization algorithm;

b) improve genetic algorithm parameter settings:
------------------------------------------------
file 'backend/optimizer/geqo/geqo_params.c', routines
'gimme_pool_size' and 'gimme_number_generations';
d233 35
a267 2
1.  optimality of the query plan
2.  computing time
d269 6
a274 23
c) find better solution for integer overflow:
---------------------------------------------
file 'backend/optimizer/geqo/geqo_eval.c', routine
'geqo_joinrel_size';
the present hack for MAXINT overflow is to set the Postgres integer
value of 'rel->size' to its logarithm;
modifications of 'struct Rel' in 'backend/nodes/relation.h' will
surely have severe impacts on the whole PostgreSQL implementation.

d) find solution for exhausted memory:
--------------------------------------
that may occur with more than 10 relations involved in a query,
file 'backend/optimizer/geqo/geqo_eval.c', routine
'gimme_tree' which is recursively called;
maybe I forgot something to be freed correctly, but I dunno what;
of course the 'rel' data structure of the JOIN keeps growing and
growing the more relations are packed into it;
suggestions are welcome :-(


2.) Further Improvements
===============================================================
Enable bushy query tree processing within PostgreSQL;
d277 53
a329 1
</ProgramListing>
d331 27
@
