/[thuban]/branches/WIP-pyshapelib-bramz/test/postgissupport.py
ViewVC logotype

Diff of /branches/WIP-pyshapelib-bramz/test/postgissupport.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1605 by bh, Tue Aug 19 11:00:40 2003 UTC revision 1947 by bh, Thu Nov 13 18:56:49 2003 UTC
# Line 115  class PostgreSQLServer: Line 115  class PostgreSQLServer:
115          self.socket_dir = socket_dir          self.socket_dir = socket_dir
116    
117          # For the client side the socket directory can be used as the          # For the client side the socket directory can be used as the
118          # host the name starts with a slash.          # host if the name starts with a slash.
119          self.host = os.path.abspath(socket_dir)          self.host = os.path.abspath(socket_dir)
120    
121            # name and password for the admin and an unprivileged user
122            self.admin_name = "postgres"
123            self.admin_password = "postgres"
124            self.user_name = "observer"
125            self.user_password = "telescope"
126    
127          # Map db names to db objects          # Map db names to db objects
128          self.known_dbs = {}          self.known_dbs = {}
129    
# Line 135  class PostgreSQLServer: Line 141  class PostgreSQLServer:
141              shutil.rmtree(self.dbdir)              shutil.rmtree(self.dbdir)
142          os.mkdir(self.dbdir)          os.mkdir(self.dbdir)
143    
144          run_command(["initdb", self.dbdir],          run_command(["initdb", "-D", self.dbdir, "-U", self.admin_name],
145                      os.path.join(self.dbdir, "initdb.log"))                      os.path.join(self.dbdir, "initdb.log"))
146    
147          extra_opts = "-p %d" % self.port          extra_opts = "-p %d" % self.port
# Line 150  class PostgreSQLServer: Line 156  class PostgreSQLServer:
156          # server ourselves          # server ourselves
157          self.wait_for_postmaster()          self.wait_for_postmaster()
158    
159            self.alter_user(self.admin_name, self.admin_password)
160            self.create_user(self.user_name, self.user_password)
161    
162      def wait_for_postmaster(self):      def wait_for_postmaster(self):
163          """Return when the database server is running          """Return when the database server is running
164    
# Line 161  class PostgreSQLServer: Line 170  class PostgreSQLServer:
170          while count < max_count:          while count < max_count:
171              try:              try:
172                  run_command(["psql", "-l", "-p", str(self.port),                  run_command(["psql", "-l", "-p", str(self.port),
173                               "-h", self.host],                               "-h", self.host, "-U", self.admin_name],
174                              os.path.join(self.dbdir, "psql-%d.log" % count))                              os.path.join(self.dbdir, "psql-%d.log" % count))
175              except:              except RuntimeError:
176                  pass                  pass
177                except:
178                    traceback.print_exc()
179              else:              else:
180                  break                  break
181              time.sleep(0.5)              time.sleep(0.5)
# Line 218  class PostgreSQLServer: Line 229  class PostgreSQLServer:
229    
230      def get_default_static_data_db(self):      def get_default_static_data_db(self):
231          dbname = "PostGISStaticTests"          dbname = "PostGISStaticTests"
232          tables = [("landmarks", os.path.join("..", "Data", "iceland",          tables = [
233                                               "cultural_landmark-point.shp")),              # Direct copies of the shapefiles. The shapeids are exactly
234                    ("political", os.path.join("..", "Data", "iceland",              # the same.
235                ("landmarks", os.path.join("..", "Data", "iceland",
236                                           "cultural_landmark-point.shp"),
237                 [("gid_offset", 1000)]),
238                ("political", os.path.join("..", "Data", "iceland",
239                                               "political.shp")),                                               "political.shp")),
240                    ("roads", os.path.join("..", "Data", "iceland",              ("roads", os.path.join("..", "Data", "iceland",
241                                           "roads-line.shp"))]                                           "roads-line.shp")),
242    
243                # The polygon data as a MULTIPOLYGON geometry type
244                ("political_multi", os.path.join("..", "Data", "iceland",
245                                                 "political.shp"),
246                 [("force_wkt_type", "MULTIPOLYGON")]),
247                ]
248          return self.get_static_data_db(dbname, tables)          return self.get_static_data_db(dbname, tables)
249    
250        def connection_params(self, user):
251            """Return the connection parameters for the given user
252    
253            The return value is a dictionary suitable as keyword argument
254            list to PostGISConnection. The user parameter may be either
255            'admin' to connect as admin or 'user' to connect as an
256            unprivileged user.
257            """
258            return {"host": self.host, "port": self.port,
259                    "user": getattr(self, user + "_name"),
260                    "password": getattr(self, user + "_password")}
261    
262        def connection_string(self, user):
263            """Return (part of) the connection string to pass to psycopg.connect
264    
265            The string contains host, port, user and password. The user
266            parameter must be either 'admin' or 'user', as for
267            connection_params.
268            """
269            params = []
270            for key, value in self.connection_params(user).items():
271                # FIXME: this doesn't do quiting correctly but that
272                # shouldn't be much of a problem (people shouldn't be using
273                # single quotes in filenames anyway :) )
274                params.append("%s='%s'" % (key, value))
275            return " ".join(params)
276    
277        def execute_sql(self, dbname, user, sql):
278            """Execute the sql statament
279    
280            The user parameter us used as in connection_params. The dbname
281            parameter must be the name of a database in the cluster.
282            """
283            conn = psycopg.connect("dbname=%s " % dbname
284                                   + self.connection_string(user))
285            cursor = conn.cursor()
286            cursor.execute(sql)
287            conn.commit()
288            conn.close()
289    
290        def require_authentication(self, required):
291            """Switch authentication requirements on or off
292    
293            When started for the first time no passwords are required. Some
294            tests want to explicitly test whether Thuban's password
295            infrastructure works and switch password authentication on
296            explicitly. When switching it on, there should be a
297            corresponding call to switch it off again in the test case'
298            tearDown method or in a finally: block.
299            """
300            if required:
301                contents = "local all password\n"
302            else:
303                contents = "local all trust\n"
304            f = open(os.path.join(self.dbdir, "pg_hba.conf"), "w")
305            f.write(contents)
306            f.close()
307            run_command(["pg_ctl", "-D", self.dbdir, "reload"],
308                        os.path.join(self.dbdir, "pg_ctl-reload.log"))
309    
310    
311        def create_user(self, username, password):
312            """Create user username with password in the database"""
313            self.execute_sql("template1", "admin",
314                             "CREATE USER %s PASSWORD '%s';" % (username,password))
315    
316        def alter_user(self, username, password):
317            """Change the user username's password in the database"""
318            self.execute_sql("template1", "admin",
319                             "ALTER USER %s PASSWORD '%s';" % (username,password))
320    
321    
322  class PostGISDatabase:  class PostGISDatabase:
# Line 234  class PostGISDatabase: Line 324  class PostGISDatabase:
324      """A PostGIS database in a PostgreSQLServer"""      """A PostGIS database in a PostgreSQLServer"""
325    
326      def __init__(self, server, postgis_sql, dbname, tables = None):      def __init__(self, server, postgis_sql, dbname, tables = None):
327            """Initialize the PostGISDatabase
328    
329            Parameters:
330    
331                server -- The PostgreSQLServer instance containing the
332                    database
333    
334                postgis_sql -- Filename of the postgis.sql file with the
335                    postgis initialization code
336    
337                dbname -- The name of the database
338    
339                tables -- Optional description of tables to create in the
340                    new database. If given it should be a list of
341                    (tablename, shapefilename) pairs meaning that a table
342                    tablename will be created with the contents of the given
343                    shapefile or (tablename, shapefilename, extraargs)
344                    triples. The extraargs should be a list of key, value
345                    pairs to use as keyword arguments to upload_shapefile.
346            """
347          self.server = server          self.server = server
348          self.postgis_sql = postgis_sql          self.postgis_sql = postgis_sql
349          self.dbname = dbname          self.dbname = dbname
# Line 243  class PostGISDatabase: Line 353  class PostGISDatabase:
353          """Remove the old db directory and create and initialize a new database          """Remove the old db directory and create and initialize a new database
354          """          """
355          run_command(["createdb", "-p", str(self.server.port),          run_command(["createdb", "-p", str(self.server.port),
356                       "-h", self.server.host, self.dbname],                       "-h", self.server.host, "-U", self.server.admin_name,
357                         self.dbname],
358                      os.path.join(self.server.dbdir, "createdb.log"))                      os.path.join(self.server.dbdir, "createdb.log"))
359          run_command(["createlang", "-p", str(self.server.port),          run_command(["createlang", "-p", str(self.server.port),
360                       "-h", self.server.host, "plpgsql", self.dbname],                       "-h", self.server.host,  "-U", self.server.admin_name,
361                         "plpgsql", self.dbname],
362                      os.path.join(self.server.dbdir, "createlang.log"))                      os.path.join(self.server.dbdir, "createlang.log"))
363          # for some reason psql doesn't exit with an error code if the          # for some reason psql doesn't exit with an error code if the
364          # file given as -f doesn't exist, so we check manually by trying          # file given as -f doesn't exist, so we check manually by trying
# Line 255  class PostGISDatabase: Line 367  class PostGISDatabase:
367          f.close()          f.close()
368          del f          del f
369          run_command(["psql", "-f", self.postgis_sql, "-d", self.dbname,          run_command(["psql", "-f", self.postgis_sql, "-d", self.dbname,
370                       "-p", str(self.server.port), "-h", self.server.host],                       "-p", str(self.server.port), "-h", self.server.host,
371                         "-U", self.server.admin_name],
372                       os.path.join(self.server.dbdir, "psql.log"))                       os.path.join(self.server.dbdir, "psql.log"))
373    
374            self.server.execute_sql(self.dbname, "admin",
375                                    "GRANT SELECT ON geometry_columns TO PUBLIC;")
376    
377          if self.tables is not None:          if self.tables is not None:
378              for tablename, shapefile in self.tables:              def unpack(item):
379                  upload_shapefile(shapefile, self, tablename)                  extra = {"force_wkt_type": None, "gid_offset": 0}
380                    if len(info) == 2:
381                        tablename, shapefile = info
382                    else:
383                        tablename, shapefile, kw = info
384                        for key, val in kw:
385                            extra[key] = val
386                    return tablename, shapefile, extra
387    
388                for info in self.tables:
389                    tablename, shapefile, kw = unpack(info)
390                    upload_shapefile(shapefile, self, tablename, **kw)
391    
392      def has_data(self, tables):      def has_data(self, tables):
393          return self.tables == tables          return self.tables == tables
# Line 322  def reason_for_not_running_tests(): Line 449  def reason_for_not_running_tests():
449         The name of the postgis_sql file is determined by find_postgis_sql()         The name of the postgis_sql file is determined by find_postgis_sql()
450       - psycopg can be imported successfully.       - psycopg can be imported successfully.
451      """      """
452        # run_command currently uses Popen4 which is not available under
453        # Windows, for example.
454        if not hasattr(popen2, "Popen4"):
455            return "Can't run PostGIS test because popen2.Popen4 does not exist"
456    
457      try:      try:
458          run_command(["pg_ctl", "--help"], None)          run_command(["pg_ctl", "--help"], None)
459      except RuntimeError:      except RuntimeError:
# Line 354  def skip_if_no_postgis(): Line 486  def skip_if_no_postgis():
486      if _cannot_run_postgis_tests:      if _cannot_run_postgis_tests:
487          raise support.SkipTest(_cannot_run_postgis_tests)          raise support.SkipTest(_cannot_run_postgis_tests)
488    
489  def point_to_wkt(coords):  def skip_if_addgeometrycolumn_does_not_use_quote_ident():
490        """Skip a test if the AddGeometryColumn function doesn't use quote_ident
491    
492        If the AddGeometryColumn function doesn't use quote_ident it doesn't
493        support unusual table or column names properly, that is, it will
494        fail with errors for names that contain spaces or double quotes.
495    
496        The test performed by this function is a bit simplistic because it
497        only tests whether the string 'quote_ident' occurs anywhere in the
498        postgis.sql file. This will hopefully works because when this was
499        fixed in postgis CVS AddGeometryColumn was the first function to use
500        quote_ident.
501        """
502        f = file(find_postgis_sql())
503        content = f.read()
504        f.close()
505        if content.find("quote_ident") < 0:
506            raise support.SkipTest("AddGeometryColumn doesn't use quote_ident")
507    
508    def coords_to_point(coords):
509      """Return string with a WKT representation of the point in coords"""      """Return string with a WKT representation of the point in coords"""
510      x, y = coords[0]      x, y = coords[0]
511      return "POINT(%r %r)" % (x, y)      return "POINT(%r %r)" % (x, y)
512    
513  def polygon_to_wkt(coords):  def coords_to_polygon(coords):
514      """Return string with a WKT representation of the polygon in coords"""      """Return string with a WKT representation of the polygon in coords"""
515      poly = []      poly = []
516      for ring in coords:      for ring in coords:
517          poly.append(", ".join(["%r %r" % p for p in ring]))          poly.append(", ".join(["%r %r" % p for p in ring]))
518      return "POLYGON((%s))" % "), (".join(poly)      return "POLYGON((%s))" % "), (".join(poly)
519    
520  def arc_to_wkt(coords):  def coords_to_multilinestring(coords):
521      """Return string with a WKT representation of the arc in coords"""      """Return string with a WKT representation of the arc in coords"""
522      poly = []      poly = []
523      for ring in coords:      for ring in coords:
524          poly.append(", ".join(["%r %r" % p for p in ring]))          poly.append(", ".join(["%r %r" % p for p in ring]))
525      return "MULTILINESTRING((%s))" % "), (".join(poly)      return "MULTILINESTRING((%s))" % "), (".join(poly)
526    
527  def upload_shapefile(filename, db, tablename):  def coords_to_multipolygon(coords):
528        """Return string with a WKT representation of the polygon in coords"""
529        poly = []
530        for ring in coords:
531            poly.append(", ".join(["%r %r" % p for p in ring]))
532        return "MULTIPOLYGON(((%s)))" % ")), ((".join(poly)
533    
534    wkt_converter = {
535        "POINT": coords_to_point,
536        "MULTILINESTRING": coords_to_multilinestring,
537        "POLYGON": coords_to_polygon,
538        "MULTIPOLYGON": coords_to_multipolygon,
539        }
540    
541    def upload_shapefile(filename, db, tablename, force_wkt_type = None,
542                         gid_offset = 0):
543        """Upload a shapefile into a new database table
544    
545        Parameters:
546    
547        filename -- The name of the shapefile
548    
549        db -- The PostGISDatabase instance representing the database
550    
551        tablename -- The name of the table to create and into which the data
552                    is to be inserted
553    
554        force_wkt_type -- If given the real WKT geometry type to use instead
555                    of the default that would be chosen based on the type of
556                    the shapefile
557    
558        gid_offset -- A number to add to the shapeid to get the value for
559                    the gid column (default 0)
560        """
561      import dbflib, shapelib      import dbflib, shapelib
562    
563        # We build this map here because we need shapelib which can only be
564        # imported after support.initthuban has been called which we can't
565        # easily do in this module because it's imported by support.
566        shp_to_wkt = {
567            shapelib.SHPT_POINT: "POINT",
568            shapelib.SHPT_ARC: "MULTILINESTRING",
569            shapelib.SHPT_POLYGON: "POLYGON",
570            }
571    
572      server = db.server      server = db.server
573      dbname = db.dbname      dbname = db.dbname
574      conn = psycopg.connect("host=%s port=%s dbname=%s"      conn = psycopg.connect("dbname=%s " % dbname
575                             % (server.host, server.port, dbname))                             + db.server.connection_string("admin"))
576      cursor = conn.cursor()      cursor = conn.cursor()
577    
578      shp = shapelib.ShapeFile(filename)      shp = shapelib.ShapeFile(filename)
# Line 400  def upload_shapefile(filename, db, table Line 593  def upload_shapefile(filename, db, table
593      #print stmt      #print stmt
594    
595      numshapes, shapetype, mins, maxs = shp.info()      numshapes, shapetype, mins, maxs = shp.info()
596      if shapetype == shapelib.SHPT_POINT:      wkttype =  shp_to_wkt[shapetype]
597          convert = point_to_wkt      if force_wkt_type:
598          wkttype = "POINT"          wkttype = force_wkt_type
599      elif shapetype == shapelib.SHPT_POLYGON:      convert = wkt_converter[wkttype]
         convert = polygon_to_wkt  
         wkttype = "POLYGON"  
     elif shapetype == shapelib.SHPT_ARC:  
         convert = arc_to_wkt  
         wkttype = "MULTILINESTRING"  
     else:  
         raise ValueError("Unsupported Shapetype %r" % shapetype)  
600    
601      cursor.execute("select AddGeometryColumn('%(dbname)s',"      cursor.execute("select AddGeometryColumn('%(dbname)s',"
602                     "'%(tablename)s', 'the_geom', '-1', '%(wkttype)s', 2);"                     "'%(tablename)s', 'the_geom', '-1', '%(wkttype)s', 2);"
# Line 424  def upload_shapefile(filename, db, table Line 610  def upload_shapefile(filename, db, table
610      for i in range(numshapes):      for i in range(numshapes):
611          data = dbf.read_record(i)          data = dbf.read_record(i)
612          data["tablename"] = tablename          data["tablename"] = tablename
613          data["gid"] = i          data["gid"] = i + gid_offset
614          data["the_geom"] = convert(shp.read_object(i).vertices())          data["the_geom"] = convert(shp.read_object(i).vertices())
615          #print insert % data          #print insert % data
616          cursor.execute(insert, data)          cursor.execute(insert, data)
617    
618        cursor.execute("GRANT SELECT ON %s TO PUBLIC;" % tablename)
619    
620      conn.commit()      conn.commit()
Legend:
Removed from v.1605  
changed lines
  Added in v.1947
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.26