/[thuban]/branches/WIP-pyshapelib-bramz/test/postgissupport.py
ViewVC logotype

Diff of /branches/WIP-pyshapelib-bramz/test/postgissupport.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2096 by bh, Thu Mar 11 13:50:53 2004 UTC revision 2717 by bernhard, Mon Jan 1 21:44:05 2007 UTC
# Line 1  Line 1 
1  # Copyright (C) 2003, 2004 by Intevation GmbH  # Copyright (C) 2003, 2004, 2005, 2006 by Intevation GmbH
2  # Authors:  # Authors:
3  # Bernhard Herzog <[email protected]>  # Bernhard Herzog <[email protected]>
4  #  #
# Line 17  import time Line 17  import time
17  import popen2  import popen2
18  import shutil  import shutil
19  import traceback  import traceback
20    import re
21    
22  import support  import support
23    
# Line 84  def run_boolean_command(command): Line 85  def run_boolean_command(command):
85  #       PostgreSQL and database  #       PostgreSQL and database
86  #  #
87    
88    _pg_ctl_command = "pg_ctl"
89    _initdb_command = "initdb"
90    # Example Values for Debian (upcomind Etch) postgresql-8.1
91    # TODO: detect which values to use here.
92    #_pg_ctl_command = "/usr/lib/postgresql/8.1/bin/pg_ctl"
93    #_initdb_command = "/usr/lib/postgresql/8.1/bin/initdb"
94    
95  class PostgreSQLServer:  class PostgreSQLServer:
96    
97      """A PostgreSQL server      """A PostgreSQL server
# Line 141  class PostgreSQLServer: Line 149  class PostgreSQLServer:
149              shutil.rmtree(self.dbdir)              shutil.rmtree(self.dbdir)
150          os.mkdir(self.dbdir)          os.mkdir(self.dbdir)
151    
152          run_command(["initdb", "-D", self.dbdir, "-U", self.admin_name],          run_command([_initdb_command, "-D", self.dbdir, "-U", self.admin_name],
153                      os.path.join(self.dbdir, "initdb.log"))                      os.path.join(self.dbdir, "initdb.log"))
154    
155          extra_opts = "-p %d" % self.port          extra_opts = "-p %d" % self.port
156          if self.socket_dir is not None:          if self.socket_dir is not None:
157              extra_opts += " -k %s" % self.socket_dir              extra_opts += " -k %s" % self.socket_dir
158          run_command(["pg_ctl", "-D", self.dbdir,          run_command([_pg_ctl_command, "-D", self.dbdir,
159                       "-l", os.path.join(self.dbdir, "logfile"),                       "-l", os.path.join(self.dbdir, "logfile"),
160                       "-o", extra_opts, "start"],                       "-o", extra_opts, "start"],
161                      os.path.join(self.dbdir, "pg_ctl-start.log"))                      os.path.join(self.dbdir, "pg_ctl-start.log"))
# Line 184  class PostgreSQLServer: Line 192  class PostgreSQLServer:
192              raise RuntimeError("postmaster didn't start")              raise RuntimeError("postmaster didn't start")
193    
194      def is_running(self):      def is_running(self):
195          """Return true a postmaster process is running on self.dbdir          """Return whether a postmaster process is running on self.dbdir
196    
197            This method runs pg_ctl status on the dbdir and returns True if
198            that command succeeds and False otherwise.
199    
200          This method runs pg_ctl status on the dbdir so even if the          Note that it is possible that this method returns true even if
201          object has just been created it is possible that this method          the PostgreSQLServer instance has just been created and
202          returns true if there's still a postmaster process running for          createdb() has not been called yet.  This can happen, for
203          self.dbdir.          instance, if the server has been started manually for debugging
204            purposes after a test suite run.
205          """          """
206          return run_boolean_command(["pg_ctl", "-D", self.dbdir, "status"])          return run_boolean_command([_pg_ctl_command, "-D", self.dbdir, "status"])
207    
208      def shutdown(self):      def shutdown(self):
209          """Stop the postmaster running for self.dbdir"""          """Stop the postmaster running for self.dbdir"""
210          run_command(["pg_ctl", "-m", "fast", "-D", self.dbdir, "stop"],          run_command([_pg_ctl_command, "-m", "fast", "-D", self.dbdir, "stop"],
211                      os.path.join(self.dbdir, "pg_ctl-stop.log"))                      os.path.join(self.dbdir, "pg_ctl-stop.log"))
212    
213      def new_postgis_db(self, dbname, tables = None, reference_systems = None):      def new_postgis_db(self, dbname, tables = None, reference_systems = None,
214                           views = None):
215          """Create and return a new PostGISDatabase object using self as server          """Create and return a new PostGISDatabase object using self as server
216          """          """
217          db = PostGISDatabase(self, self.postgis_sql, dbname, tables = tables,          db = PostGISDatabase(self, self.postgis_sql, dbname, tables = tables,
218                               reference_systems = reference_systems)                               reference_systems = reference_systems,
219                                 views = views)
220          db.initdb()          db.initdb()
221          self.known_dbs[dbname] = db          self.known_dbs[dbname] = db
222          return db          return db
223    
224      def get_static_data_db(self, dbname, tables, reference_systems):      def get_static_data_db(self, dbname, tables, reference_systems, views):
225          """Return a PostGISDatabase for a database with the given static data          """Return a PostGISDatabase for a database with the given static data
226    
227          If no databasse of the name dbname exists, create a new one via          If no databasse of the name dbname exists, create a new one via
# Line 225  class PostgreSQLServer: Line 239  class PostgreSQLServer:
239          """          """
240          db = self.known_dbs.get(dbname)          db = self.known_dbs.get(dbname)
241          if db is not None:          if db is not None:
242              if db.has_data(tables, reference_systems):              if db.has_data(tables, reference_systems, views):
243                  return db                  return db
244              raise ValueError("PostGISDatabase named %r doesn't have tables %r"              raise ValueError("PostGISDatabase named %r doesn't have tables %r"
245                               % (dbname, tables))                               % (dbname, tables))
246          return self.new_postgis_db(dbname, tables, reference_systems)          return self.new_postgis_db(dbname, tables, reference_systems, views)
247    
248      def get_default_static_data_db(self):      def get_default_static_data_db(self):
249          dbname = "PostGISStaticTests"          dbname = "PostGISStaticTests"
# Line 244  class PostgreSQLServer: Line 258  class PostgreSQLServer:
258               [("gid_offset", 1000)]),               [("gid_offset", 1000)]),
259              ("political", os.path.join("..", "Data", "iceland",              ("political", os.path.join("..", "Data", "iceland",
260                                               "political.shp")),                                               "political.shp")),
261              ("roads", os.path.join("..", "Data", "iceland",              ("roads_multi", os.path.join("..", "Data", "iceland",
262                                           "roads-line.shp")),                                           "roads-line.shp")),
263    
264                # same as roads-multi but using LINESTRING instead of
265                # MULTILINESTRING
266                ("roads", os.path.join("..", "Data", "iceland",
267                                             "roads-line.shp"),
268                 [("force_wkt_type", "LINESTRING")]),
269    
270              # The polygon data as a MULTIPOLYGON geometry type              # The polygon data as a MULTIPOLYGON geometry type
271              ("political_multi", os.path.join("..", "Data", "iceland",              ("political_multi", os.path.join("..", "Data", "iceland",
272                                               "political.shp"),                                               "political.shp"),
# Line 266  class PostgreSQLServer: Line 286  class PostgreSQLServer:
286                ("srid", 1),                ("srid", 1),
287                ("gid_column", "point_id")]),                ("gid_column", "point_id")]),
288              ]              ]
289          return self.get_static_data_db(dbname, tables, srids)          views = [("v_landmarks", "SELECT * FROM landmarks_point_id")]
290            return self.get_static_data_db(dbname, tables, srids, views)
291    
292      def connection_params(self, user):      def connection_params(self, user):
293          """Return the connection parameters for the given user          """Return the connection parameters for the given user
# Line 296  class PostgreSQLServer: Line 317  class PostgreSQLServer:
317          return " ".join(params)          return " ".join(params)
318    
319      def execute_sql(self, dbname, user, sql):      def execute_sql(self, dbname, user, sql):
320          """Execute the sql statament          """Execute the sql statament and return a result for SELECT statements
321    
322          The user parameter us used as in connection_params. The dbname          The user parameter us used as in connection_params. The dbname
323          parameter must be the name of a database in the cluster.          parameter must be the name of a database in the cluster.  The
324            sql parameter is the SQL statement to execute as a string.  If
325            the string starts with 'select' (matched case insensitively) the
326            first row of the result will be returned.  Otherwise the return
327            value is None.
328          """          """
329          conn = psycopg.connect("dbname=%s " % dbname          conn = psycopg.connect("dbname=%s " % dbname
330                                 + self.connection_string(user))                                 + self.connection_string(user))
331          cursor = conn.cursor()          cursor = conn.cursor()
332          cursor.execute(sql)          cursor.execute(sql)
333            if sql.lower().startswith("select"):
334                row = cursor.fetchone()
335            else:
336                row = None
337          conn.commit()          conn.commit()
338          conn.close()          conn.close()
339            return row
340    
341        def server_version(self):
342            """Return the server version as a tuple (major, minor, patch)
343    
344            Each item in the tuple is an int.
345            """
346            result = self.execute_sql("template1", "admin", "SELECT version();")[0]
347            match = re.match(r"PostgreSQL (\d+\.\d+\.\d+)", result)
348            if match:
349                return tuple(map(int, match.group(1).split(".")))
350            else:
351                raise RutimeError("Cannot determine PostgreSQL server version"
352                                  " from %r" % result)
353    
354      def require_authentication(self, required):      def require_authentication(self, required):
355          """Switch authentication requirements on or off          """Switch authentication requirements on or off
# Line 318  class PostgreSQLServer: Line 361  class PostgreSQLServer:
361          corresponding call to switch it off again in the test case'          corresponding call to switch it off again in the test case'
362          tearDown method or in a finally: block.          tearDown method or in a finally: block.
363          """          """
364            # Starting with PostgreSQL 7.3 the pg_hba.conf file has an
365            # additional column with a username.  Query the server version
366            # and generate a file in the correct format.
367            if self.server_version() >= (7, 3):
368                user = "all"
369            else:
370                user = ""
371          if required:          if required:
372              contents = "local all password\n"              contents = "local all %s password\n" % user
373          else:          else:
374              contents = "local all trust\n"              contents = "local all %s trust\n" % user
375          f = open(os.path.join(self.dbdir, "pg_hba.conf"), "w")          f = open(os.path.join(self.dbdir, "pg_hba.conf"), "w")
376          f.write(contents)          f.write(contents)
377          f.close()          f.close()
378          run_command(["pg_ctl", "-D", self.dbdir, "reload"],          run_command([_pg_ctl_command, "-D", self.dbdir, "reload"],
379                      os.path.join(self.dbdir, "pg_ctl-reload.log"))                      os.path.join(self.dbdir, "pg_ctl-reload.log"))
380    
381    
# Line 345  class PostGISDatabase: Line 395  class PostGISDatabase:
395      """A PostGIS database in a PostgreSQLServer"""      """A PostGIS database in a PostgreSQLServer"""
396    
397      def __init__(self, server, postgis_sql, dbname, tables = None,      def __init__(self, server, postgis_sql, dbname, tables = None,
398                   reference_systems = ()):                   reference_systems = (), views = None):
399          """Initialize the PostGISDatabase          """Initialize the PostGISDatabase
400    
401          Parameters:          Parameters:
# Line 353  class PostGISDatabase: Line 403  class PostGISDatabase:
403              server -- The PostgreSQLServer instance containing the              server -- The PostgreSQLServer instance containing the
404                  database                  database
405    
406              postgis_sql -- Filename of the postgis.sql file with the              postgis_sql -- Filename of the sql file with the postgis
407                  postgis initialization code                  initialization code
408    
409              dbname -- The name of the database              dbname -- The name of the database
410    
# Line 371  class PostGISDatabase: Line 421  class PostGISDatabase:
421                  (srid, params) pairs where srid is the srid defined by                  (srid, params) pairs where srid is the srid defined by
422                  the proj4 paramter string params.  The srid can be given                  the proj4 paramter string params.  The srid can be given
423                  as an extra parameter in the tables list.                  as an extra parameter in the tables list.
424    
425                views -- Optional description of views.  If given it should
426                    be a list of (viewname, select_stmt) pairs where
427                    viewname is the name of the view to be created and
428                    select_stmt is the select statement to use as the basis.
429                    The views will be created after the tables and may refer
430                    to them in the select_stmt.
431          """          """
432          self.server = server          self.server = server
433          self.postgis_sql = postgis_sql          self.postgis_sql = postgis_sql
434          self.dbname = dbname          self.dbname = dbname
435          self.tables = tables          self.tables = tables
436            self.views = views
437          if reference_systems:          if reference_systems:
438              self.reference_systems = reference_systems              self.reference_systems = reference_systems
439          else:          else:
# Line 431  class PostGISDatabase: Line 489  class PostGISDatabase:
489                  tablename, shapefile, kw = unpack(info)                  tablename, shapefile, kw = unpack(info)
490                  upload_shapefile(shapefile, self, tablename, **kw)                  upload_shapefile(shapefile, self, tablename, **kw)
491    
492      def has_data(self, tables, reference_systems):          if self.views is not None:
493                for viewname, select_stmt in self.views:
494                    self.server.execute_sql(self.dbname, "admin",
495                                            "CREATE VIEW %s AS %s" % (viewname,
496                                                                      select_stmt))
497                    self.server.execute_sql(self.dbname, "admin",
498                                            "GRANT SELECT ON %s TO PUBLIC;"
499                                            % viewname)
500    
501        def has_data(self, tables, reference_systems, views):
502          return (self.tables == tables          return (self.tables == tables
503                  and self.reference_systems == reference_systems)                  and self.reference_systems == reference_systems
504                    and self.views == views)
505    
506    
507  def find_postgis_sql():  def find_postgis_sql():
508      """Return the name of the postgis_sql file      """Return the name of the postgis_sql file
509    
510      A postgis installation usually has the postgis_sql file in      A postgis installation usually has the postgis_sql file in
511      PostgreSQL's datadir (i.e. the directory where PostgreSQL keeps      PostgreSQL's $datadir (i.e. the directory where PostgreSQL keeps
512      static files, not the directory containing the databases).      static files, not the directory containing the databases).
513      Unfortunately there's no way to determine the name of this directory      Unfortunately there's no way to determine the name of this directory
514      with pg_config so we assume here that it's      with pg_config so we assume here that it's
515      $bindir/../share/postgresql/.      $bindir/../share/postgresql/.
516    
517        Furthermore, different versions of postgis place the file in
518        slightly different locations or may even use different names.  For
519        instance:
520    
521          postgis 0.7.5        $datadir/contrib/postgis.sql
522          postgis 0.8.1        $datadir/postgis.sql
523          postgis 1.0.0-rc1    $datadir/lwpostgis.sql
524          postgis 1.0.0-rc4    $datadir/contrib/lwpostgis.sql
525    
526        To support both versions, we look in both places and return the
527        first one found (looking under contrib first).  
528    
529        Debian (umcoming Etch) can do several version of postgresql
530        and thus has changed the paths. We try one location
531        in datadir2 only for Debian Etch postgresql-8.1.
532    
533        If the file is not found the return value is None.
534      """      """
535      bindir = run_config_script("pg_config --bindir").strip()      bindir = run_config_script("pg_config --bindir").strip()
536      return os.path.join(bindir, "..", "share", "postgresql",      datadir = os.path.join(bindir, "..", "share", "postgresql")
537                          "contrib", "postgis.sql")      datadir2 = os.path.join("/", "usr", "share", "postgresql-8.1-postgis")
538    
539        for filename in [os.path.join(datadir, "contrib", "postgis.sql"),
540                         os.path.join(datadir, "postgis.sql"),
541                         os.path.join(datadir, "lwpostgis.sql"),
542                         os.path.join(datadir, "contrib", "lwpostgis.sql"),
543                         os.path.join(datadir2, "lwpostgis.sql") \
544                        ]:
545            if os.path.exists(filename):
546                return filename
547    
548    
549  _postgres_server = None  _postgres_server = None
550  def get_test_server():  def get_test_server():
# Line 498  def reason_for_not_running_tests(): Line 594  def reason_for_not_running_tests():
594          return "Can't run PostGIS test because popen2.Popen4 does not exist"          return "Can't run PostGIS test because popen2.Popen4 does not exist"
595    
596      try:      try:
597          run_command(["pg_ctl", "--help"], None)          run_command([_pg_ctl_command, "--help"], None)
598      except RuntimeError:      except RuntimeError:
599          return "Can't run PostGIS tests because pg_ctl fails"          return "Can't run PostGIS tests because pg_ctl fails"
600    
# Line 560  def coords_to_polygon(coords): Line 656  def coords_to_polygon(coords):
656          poly.append(", ".join(["%r %r" % p for p in ring]))          poly.append(", ".join(["%r %r" % p for p in ring]))
657      return "POLYGON((%s))" % "), (".join(poly)      return "POLYGON((%s))" % "), (".join(poly)
658    
659    def coords_to_linestring(coords):
660        """Return string with a LINESTRING WKT representation of coords"""
661        if len(coords) > 1:
662            raise ValueError("A LINESTRING can only have one arc")
663        return "LINESTRING(%s)" % ", ".join(["%r %r" % p for p in coords[0]])
664    
665  def coords_to_multilinestring(coords):  def coords_to_multilinestring(coords):
666      """Return string with a WKT representation of the arc in coords"""      """Return string with a MULTILINESTRING WKT representation of coords"""
667      poly = []      poly = []
668      for ring in coords:      for ring in coords:
669          poly.append(", ".join(["%r %r" % p for p in ring]))          poly.append(", ".join(["%r %r" % p for p in ring]))
# Line 576  def coords_to_multipolygon(coords): Line 678  def coords_to_multipolygon(coords):
678    
679  wkt_converter = {  wkt_converter = {
680      "POINT": coords_to_point,      "POINT": coords_to_point,
681        "LINESTRING": coords_to_linestring,
682      "MULTILINESTRING": coords_to_multilinestring,      "MULTILINESTRING": coords_to_multilinestring,
683      "POLYGON": coords_to_polygon,      "POLYGON": coords_to_polygon,
684      "MULTIPOLYGON": coords_to_multipolygon,      "MULTIPOLYGON": coords_to_multipolygon,
# Line 594  def upload_shapefile(filename, db, table Line 697  def upload_shapefile(filename, db, table
697      tablename -- The name of the table to create and into which the data      tablename -- The name of the table to create and into which the data
698                  is to be inserted                  is to be inserted
699    
700      force_wkt_type -- If given the real WKT geometry type to use instead      force_wkt_type -- If given and not None, this is used as the WKT
701                  of the default that would be chosen based on the type of                  geometry type to use instead of the default that would
702                  the shapefile                  be chosen based on the type of the shapefile
703    
704      gid_offset -- A number to add to the shapeid to get the value for      gid_offset -- A number to add to the shapeid to get the value for
705                  the gid column (default 0)                  the gid column (default 0)
# Line 609  def upload_shapefile(filename, db, table Line 712  def upload_shapefile(filename, db, table
712    
713      srid -- The srid of the spatial references system used by the table      srid -- The srid of the spatial references system used by the table
714              and the data              and the data
715    
716        The tables will be explicitely created WITH OIDS. This has been
717        default for PostgreSQL <8.0 and some tests relied on it (end of 2006).
718      """      """
719      import dbflib, shapelib      import dbflib, shapelib
720    
# Line 647  def upload_shapefile(filename, db, table Line 753  def upload_shapefile(filename, db, table
753          fields.append(name)          fields.append(name)
754          fields_decl.append("%s %s" % (name, typemap[ftype]))          fields_decl.append("%s %s" % (name, typemap[ftype]))
755          insert_formats.append("%%(%s)s" % name)          insert_formats.append("%%(%s)s" % name)
756      stmt = "CREATE TABLE %s (\n    %s\n);" % (tablename,      stmt = "CREATE TABLE %s (\n    %s\n) WITH OIDS ;" % (tablename,
757                                                ",\n    ".join(fields_decl))                                                ",\n    ".join(fields_decl))
758      cursor.execute(stmt)      cursor.execute(stmt)
759      #print stmt      #print stmt
Legend:
Removed from v.2096  
changed lines
  Added in v.2717
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.26