/[thuban]/branches/WIP-pyshapelib-bramz/Thuban/Model/transientdb.py
ViewVC logotype

Annotation of /branches/WIP-pyshapelib-bramz/Thuban/Model/transientdb.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 841 - (hide annotations)
Tue May 6 17:12:53 2003 UTC (21 years, 10 months ago) by bh
Original Path: trunk/thuban/Thuban/Model/transientdb.py
File MIME type: text/x-python
File size: 14633 byte(s)
(TransientTableBase.SimpleQuery):
Implement a simple query method for the query dialog
(TransientTableBase.create): Add an INTEGER PRIMARY KEY that holds
the row index or shapeid.
(TransientTable.create): Insert the right value of the row index
(TransientJoinedTable.create): Copy the row index of the left
table to the joined result table

1 bh 765 # Copyright (C) 2003 by Intevation GmbH
2     # Authors:
3     # Bernhard Herzog <[email protected]>
4     #
5     # This program is free software under the GPL (>=v2)
6     # Read the file COPYING coming with the software for details.
7    
8     """Database for transient data
9    
10     This database is intended for data representations needed during the
11     lifetime of a Thuban session but which is not permanent. Examples of
12     this are for instance a join of two DBF files where the DBF files are
13     the permanent representation of the data and the join only exists in the
14     Thuban session and is reconstructed when the session is opened.
15     """
16    
17     __version__ = "$Revision$"
18     # $Source$
19     # $Id$
20    
21     import os
22     import weakref
23     from sqlite import connect
24    
25     import table
26    
27     sql_type_map = {
28     table.FIELDTYPE_INT: "INTEGER",
29     table.FIELDTYPE_STRING: "VARCHAR",
30     table.FIELDTYPE_DOUBLE: "FLOAT",
31     }
32    
33     type_converter_map = {
34     table.FIELDTYPE_INT: int,
35     table.FIELDTYPE_STRING: str,
36     table.FIELDTYPE_DOUBLE: float,
37     }
38    
39     class TransientDatabase:
40    
41     def __init__(self, filename):
42     self.filename = filename
43     self.conn = connect(filename)
44     # Counters to produce unique table and column names
45     self.num_tables = 0
46     self.num_cols = 0
47     # Since there's only once process using the SQLite database, we
48     # might be able to get a tad more speed with default_synchronous
49     # OFF. So far I haven't seen any measurable speedup, though.
50     #self.execute("PRAGMA default_synchronous = OFF")
51    
52     def __del__(self):
53     self.close()
54    
55     def close(self):
56 bh 777 if self.conn is not None:
57 bh 765 self.conn.close()
58 bh 777 self.conn = None
59 bh 765
60     def new_table_name(self):
61     self.num_tables += 1
62     return "Table%03d" % self.num_tables
63    
64     def new_column_name(self):
65     self.num_cols += 1
66     return "Col%03d" % self.num_cols
67    
68     def execute(self, *args):
69     """execute the SQL statement in the database and return the result"""
70     cursor = self.conn.cursor()
71     cursor.execute(*args)
72     result = cursor.fetchone()
73     self.conn.commit()
74     return result
75    
76     def cursor(self):
77     return self.conn.cursor()
78    
79    
80     class ColumnReference:
81    
82     def __init__(self, name, type, internal_name):
83     self.name = name
84     self.type = type
85     self.internal_name = internal_name
86    
87    
88 bh 818 class TransientTableBase(table.OldTableInterfaceMixin):
89 bh 765
90     """Base class for tables in the transient database"""
91    
92     def __init__(self, transient_db):
93     """Initialize the table for use with the given transient db"""
94     self.db = transient_db
95     self.tablename = self.db.new_table_name()
96     self.indexed_columns = {}
97     self.read_record_cursor = None
98     self.read_record_last_row = None
99 bh 785 self.read_record_last_result = None
100 bh 765
101     def create(self, columns):
102     self.columns = columns
103     self.name_to_column = {}
104     self.orig_names = []
105     self.internal_to_orig = {}
106     self.orig_to_internal = {}
107 bh 818 self.column_map = {}
108 bh 765
109     # Create the column objects and fill various maps and lists
110 bh 818 for index in range(len(self.columns)):
111     col = self.columns[index]
112 bh 765 self.name_to_column[col.name] = col
113     self.orig_names.append(col.name)
114     self.internal_to_orig[col.internal_name] = col.name
115     self.orig_to_internal[col.name] = col.internal_name
116 bh 818 self.column_map[col.name] = col
117     self.column_map[index] = col
118 bh 765
119     # Build the CREATE TABLE statement and create the table in the
120     # database
121 bh 841 table_types = ["id INTEGER PRIMARY KEY"]
122 bh 765 for col in self.columns:
123     table_types.append("%s %s" % (col.internal_name,
124     sql_type_map[col.type]))
125     table_stmt = "CREATE TABLE %s (\n %s\n);" % (self.tablename,
126     ",\n ".join(table_types))
127     self.db.execute(table_stmt)
128    
129     def transient_table(self):
130     """
131     Return a table whose underlying implementation is in the transient db
132     """
133     return self
134    
135     def ensure_index(self, column):
136     """Ensure that there's an index on the given column"""
137     if not column in self.indexed_columns:
138     internal_name = self.orig_to_internal[column]
139     indexname = "Index_%s_%s" % (self.tablename, internal_name)
140     stmt = "CREATE INDEX %s ON %s (%s);" % (indexname, self.tablename,
141     internal_name)
142     self.db.execute(stmt)
143     self.indexed_columns[column] = 1
144    
145 bh 818 def NumColumns(self):
146 bh 765 return len(self.columns)
147    
148 bh 818 def NumRows(self):
149 bh 765 result = self.db.execute("SELECT count(*) FROM %s;" % self.tablename)
150     return int(result[0])
151    
152 bh 818 def Columns(self):
153     return self.columns
154    
155     def Column(self, col):
156     return self.column_map[col]
157    
158 bh 839 def HasColumn(self, col):
159     """Return whether the table has a column with the given name or index
160     """
161     return self.column_map.has_key(col)
162    
163 bh 818 def ReadRowAsDict(self, index):
164 bh 765 if self.read_record_cursor is None or index <self.read_record_last_row:
165     stmt = ("SELECT %s FROM %s;"
166     % (", ".join([c.internal_name for c in self.columns]),
167     self.tablename))
168     self.read_record_cursor = self.db.cursor()
169     self.read_record_cursor.execute(stmt)
170     self.read_record_last_row = -1
171 bh 785 self.read_record_last_result = None
172    
173     # Now we should have a cursor at a position less than or equal
174     # to the index so the following if statement will always set
175     # result to a suitable value
176     assert index >= self.read_record_last_row
177    
178     if index == self.read_record_last_row:
179 bh 818 result = self.read_record_last_result
180 bh 785 else:
181     for i in range(index - self.read_record_last_row):
182     result = self.read_record_cursor.fetchone()
183     self.read_record_last_result = result
184 bh 765 self.read_record_last_row = index
185 bh 785 return dict(zip(self.orig_names, result))
186 bh 765
187 bh 818 def ValueRange(self, col):
188     col = self.column_map[col]
189 bh 765 iname = col.internal_name
190     min, max = self.db.execute("SELECT min(%s), max(%s) FROM %s;"
191     % (iname, iname, self.tablename))
192     converter = type_converter_map[col.type]
193 bh 818 return (converter(min), converter(max))
194 bh 765
195 bh 818 def UniqueValues(self, col):
196     iname = self.column_map[col].internal_name
197 bh 765 cursor = self.db.cursor()
198     cursor.execute("SELECT %s FROM %s GROUP BY %s;"
199     % (iname, self.tablename, iname))
200     result = []
201     while 1:
202     row = cursor.fetchone()
203     if row is None:
204     break
205     result.append(row[0])
206     return result
207    
208 bh 841 def SimpleQuery(self, left, comparison, right):
209     """Return the indices of all rows that matching a condition.
210 bh 765
211 bh 841 Parameters:
212     left -- The column object for the left side of the comparison
213    
214     comparison -- The comparison operator as a string. It must be
215     one of '==', '!=', '<', '<=', '>=', '>'
216    
217     right -- The right hand side of the comparison. It must be
218     either a column object or a value, i.e. a string,
219     int or float.
220    
221     The return value is a sorted list of the indices of the rows
222     where the condition is true.
223     """
224     if comparison not in ("==", "!=", "<", "<=", ">=", ">"):
225     raise ValueError("Comparison operator %r not allowed" % comparison)
226    
227     if hasattr(right, "internal_name"):
228     right_template = right.internal_name
229     params = ()
230     else:
231     right_template = "%s"
232     params = (right,)
233    
234     query = "SELECT id FROM %s WHERE %s %s %s ORDER BY id;" \
235     % (self.tablename, left.internal_name, comparison,
236     right_template)
237    
238     cursor = self.db.cursor()
239     cursor.execute(query, params)
240     result = []
241     while 1:
242     row = cursor.fetchone()
243     if row is None:
244     break
245     result.append(row[0])
246     return result
247    
248    
249 bh 765 class TransientTable(TransientTableBase):
250    
251     """A Table in a transient DB that starts as the copy of a Thuban Table."""
252    
253     def __init__(self, transient_db, table):
254     """Create a new table in the given transient DB as a copy of table
255    
256     The table argument can be any object implementing the Table
257     interface.
258     """
259     TransientTableBase.__init__(self, transient_db)
260     self.create(table)
261    
262     def create(self, table):
263     columns = []
264 bh 818 for col in table.Columns():
265     columns.append(ColumnReference(col.name, col.type,
266 bh 765 self.db.new_column_name()))
267     TransientTableBase.create(self, columns)
268    
269     # copy the input table to the transient db
270 bh 841
271     # A key to insert to use for the formatting of the insert
272     # statement. The key must not be equal to any of the column
273     # names so we construct one by building a string of x's that is
274     # longer than any of the column names
275     id_key = max([len(col.name) for col in self.columns]) * "x"
276    
277     insert_template = "INSERT INTO %s (id, %s) VALUES (%%(%s)s, %s);" \
278 bh 765 % (self.tablename,
279     ", ".join([col.internal_name
280     for col in self.columns]),
281 bh 841 id_key,
282 bh 765 ", ".join(["%%(%s)s" % col.name
283     for col in self.columns]))
284     cursor = self.db.cursor()
285 bh 818 for i in range(table.NumRows()):
286 bh 841 row = table.ReadRowAsDict(i)
287     row[id_key] = i
288     cursor.execute(insert_template, row)
289 bh 765 self.db.conn.commit()
290    
291    
292    
293     class TransientJoinedTable(TransientTableBase):
294    
295     """A Table in the transient DB that contains a join of two tables"""
296    
297     def __init__(self, transient_db, left_table, left_field,
298     right_table, right_field = None):
299     """Create a new table in the transient DB as a join of two tables.
300    
301     Both input tables, left_table and right_table must have a
302     transient_table method that returns a table object for a table
303     in the trnsient database. The join is performed on the condition
304     that the value of the left_field column the the left table is
305     equal to the value of the right_field in the right_table.
306    
307     The joined table contains all columns of the input tables with
308     one exception: Any column in the right_table with the same name
309     as one of the columns in the left_table will be omitted. This is
310     somewhat of an implementation detail, but is done so that the
311     column names of the joined table can be the same as the column
312     names of the input tables without having to create prefixes.
313     """
314     TransientTableBase.__init__(self, transient_db)
315     self.left_table = left_table.transient_table()
316     self.left_field = left_field
317     self.right_table = right_table.transient_table()
318     if right_field:
319     self.right_field = right_field
320     else:
321     self.right_field = self.left_field
322     self.create()
323    
324     def create(self):
325     """Internal: Create the table with the joined data"""
326     self.tablename = self.db.new_table_name()
327    
328     self.right_table.ensure_index(self.right_field)
329    
330     # Coalesce the column information
331     visited = {}
332     columns = []
333     for col in self.left_table.columns + self.right_table.columns:
334     if col.name in visited:
335 bh 841 # We can't allow multiple columns with the same original
336     # name, so omit this one. FIXME: There should be a
337     # better solution.
338 bh 765 continue
339     columns.append(col)
340     TransientTableBase.create(self, columns)
341    
342     # Copy the joined data to the table.
343     internal_names = [col.internal_name for col in self.columns]
344 bh 841 stmt = ("INSERT INTO %s (id, %s) SELECT %s.id, %s FROM %s"
345     " JOIN %s ON %s = %s;"
346     % (self.tablename,
347     ", ".join(internal_names),
348     self.left_table.tablename,
349     ", ".join(internal_names),
350     self.left_table.tablename,
351     self.right_table.tablename,
352     self.orig_to_internal[self.left_field],
353     self.orig_to_internal[self.right_field]))
354 bh 765 self.db.execute(stmt)
355    
356    
357 bh 818 class AutoTransientTable(table.OldTableInterfaceMixin):
358 bh 765
359     """Table that copies data to a transient table on demand.
360    
361     The AutoTransientTable takes another table as input and copies data
362     to a table in a TransientDatabase instance on demand.
363     """
364    
365     def __init__(self, transient_db, table):
366     self.transient_db = transient_db
367     self.table = table
368     self.t_table = None
369    
370 bh 818 def Columns(self):
371     return self.table.Columns()
372 bh 765
373 bh 818 def Column(self, col):
374     return self.table.Column(col)
375 bh 765
376 bh 839 def HasColumn(self, col):
377     """Return whether the table has a column with the given name or index
378     """
379     return self.table.HasColumn(col)
380    
381 bh 818 def NumRows(self):
382     return self.table.NumRows()
383 bh 765
384 bh 818 def NumColumns(self):
385     return self.table.NumColumns()
386 bh 765
387 bh 818 def ReadRowAsDict(self, record):
388 bh 765 """Return the record no. record as a dict mapping field names to values
389     """
390     if self.t_table is not None:
391 bh 839 return self.t_table.ReadRowAsDict(record)
392 bh 765 else:
393 bh 818 return self.table.ReadRowAsDict(record)
394 bh 765
395     def copy_to_transient(self):
396     """Internal: Create a transient table and copy the data into it"""
397     self.t_table = TransientTable(self.transient_db, self)
398    
399     def transient_table(self):
400     """
401     Return a table whose underlying implementation is in the transient db
402     """
403     if self.t_table is None:
404     self.copy_to_transient()
405     return self.t_table
406    
407 bh 818 def ValueRange(self, col):
408 bh 765 if self.t_table is None:
409     self.copy_to_transient()
410 bh 839 return self.t_table.ValueRange(col)
411 bh 765
412 bh 839 def UniqueValues(self, col):
413 bh 765 if self.t_table is None:
414     self.copy_to_transient()
415 bh 839 return self.t_table.UniqueValues(col)

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision

[email protected]
ViewVC Help
Powered by ViewVC 1.1.26