data/xulugridfile/XuluGridFile.java

package appl.parallel.data.xulugridfile;

import java.awt.Rectangle;
import java.awt.image.DataBuffer;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.RandomAccessFile;

import appl.parallel.data.WritableGridArrayPartition;
import appl.parallel.spmd.split.SplittableResource;
import appl.parallel.spmd.split.WritableGridPartition;
import de.appl.util.RasterMetaData;
import de.schmitzm.geotools.data.WritableGrid;

/**
 * A XuluGridfile is a memory- and 2D-access optimized file format
 * which can be used for parallel programming. All data is accessed
 * on disk only (and not loaded into memory).
 * For this the data is stored in a special uncompressed format.<p/>
 *
 * The values are stored line by line in the GridFile. The result is, that
 * every value is stored at a well known position in the file,
 * so that the position of the value in the file can be calculated.
 * Because of this it is not necessary to search the file for a value
 * of a given coordinate.<br>
 * If a partition is requested it is possible to read the data directly from
 * the file.<p/>
 *
 * The filename of the XuluGridFile must end with ".xgrid"
 * The metadata of the gird is stored in a separate file with the same name in
 * the same directory ending with ".xworld".<p/>
 *
 * The XuluGridFile is buffered, it can handle Float, Double and Int values.
 * To create a XuluGridFile you can use {@link #XuluGridFile(File, RasterMetaData) this} constructor, or
 * you can convert an existing Grid into the {@link XuluGridFile} format using
 * the {@link XuluGridFileConverter}.<p/>
 *
 * A monitor is used to guarantee that only one XuluGridFile is accessed at a time.
 * This guarantees a high disk performance.<p/>
 *
 * For use in the XuluPlatform you can use the {@link XuluWritableGridFile}, which
 * implements the {@link WritableGrid} interface.<p/>
 *
 *
 * @author Dominik Appl
 *
 */
public class XuluGridFile {

        protected RandomAccessFile gridFile = null;

        protected RasterMetaData metaData = null;

        //length of one datavalue in bytes
        private int dataValueSize = 4;

        //buffer of 256 kb
        private int buffersize = 262144;

        private final File outputFile;

        //the monitor is used to guarantee that only one XuluGridFile is accessed at a time
        private static Object monitor = new Object();

        /**
         * Opens an <b>existing</b> Xulu Grid File. The File must have the ending ".xgrid" and there
         * must be a xuluGrid world file in the same directory with the ending ".xworld"
         *
         * @param file the filename to read from and write to
         * @param mode the input mode as specified at {@link RandomAccessFile#RandomAccessFile(java.lang.String, java.lang.String)}
         * <blockquote><table summary="Access mode permitted values and meanings">
         *<tr><th><p align="left">Value</p></th><th><p align="left">Meaning</p></th></tr>
         *<tr><td valign="top"><tt>"r"</tt></td>
         *    <td> Open for reading only.  Invoking any of the <tt>write</tt>
         *    methods of the resulting object will cause an <A HREF="../../java/io/IOException.html" title="class in java.io"><CODE>IOException</CODE></A> to be thrown. </td></tr>
         *
         *<tr><td valign="top"><tt>"rw"</tt></td>
         *    <td> Open for reading and writing.  If the file does not already
         *    exist then an attempt will be made to create it. </td></tr>
         *<tr><td valign="top"><tt>"rws"</tt></td>
         *    <td> Open for reading and writing, as with <tt>"rw"</tt>, and also
         *    require that every update to the file's content or metadata be
         *    written synchronously to the underlying storage device.  </td></tr>
         *<tr><td valign="top"><tt>"rwd"&nbsp;&nbsp;</tt></td>
         *     <td> Open for reading and writing, as with <tt>"rw"</tt>, and also
         *    require that every update to the file's content be written
         *    synchronously to the underlying storage device. </td></tr>
         *</table></blockquote>
         *
         * @throws FileNotFoundException
         * @see RandomAccessFile
         * */
        public XuluGridFile(File file, String mode) throws FileNotFoundException,
                        XuluGridFileException {
                outputFile = file;
                checkFileName(file);
                try {
                        //gridFile = new BufferedRandomAccessFile(file.getAbsolutePath(), mode, buffersize);
                        gridFile = new RandomAccessFile(file.getAbsolutePath(), mode);
                } catch (IOException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }
                String worldFileName = file.getAbsolutePath().replaceAll(".xgrid$",
                                ".xworld");
                this.metaData = readXuluGridWorldFile(new File(worldFileName));
        }

        /**
         * Opens an<b> existing</b> gridfile in read/write mode.
         *
         * @param file the existing XuluGridFile to be opened
         * @throws FileNotFoundException
         * @throws XuluGridFileException
         */
        public XuluGridFile(File file) throws FileNotFoundException,
                        XuluGridFileException {
                this(file, "rw");
        }

        /**
         * Creates a <b>NEW</b> File with the sample given as {@link RasterMetaData}
         * @param targetFile the file the new grid will be written to. <b>Has to end with ".xgrid"!</b>
         * @param metaData the metadata of the new grid. Width and height are used to allocate the space for the new XGrid.
         * @throws XuluGridFileException
         */
        public XuluGridFile(File targetFile, RasterMetaData metaData)
                        throws XuluGridFileException {
                outputFile = targetFile;
                this.metaData = metaData;
                checkFileName(targetFile);
                writeWorldFileForMetaData(targetFile, metaData);
                setByteLength(metaData);
                try {
                        //gridFile = new BufferedRandomAccessFile(targetFile.getAbsolutePath(), "rw",buffersize );
                        gridFile = new RandomAccessFile(targetFile.getAbsolutePath(), "rw");
                        //allocate space by extending the file
                        gridFile.setLength(metaData.getWidth() * metaData.getHeight()
                                        * dataValueSize);
                } catch (FileNotFoundException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                } catch (IOException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }

        }

        /**
         * Sets the bytelength (4 bytes for float/int, 8 for double)
         *
         * @param meta the metadata object of the grid
         * @throws XuluGridFileException
         */
        private void setByteLength(RasterMetaData meta)
                        throws XuluGridFileException {
                switch (metaData.getDataType()) {
                case DataBuffer.TYPE_DOUBLE:
                        dataValueSize = 8;
                        break;
                case DataBuffer.TYPE_FLOAT:
                case DataBuffer.TYPE_INT:
                        dataValueSize = 4;
                        break;
                default:
                        throw new XuluGridFileException(
                                        "XuluGridFile supports only float,double and int Datatypes!");
                }
        }

        /**
         * Creates a new XuluGridFile out of an existing Writable Grid and opens it in read-write mode. The {@link WritableGrid}-Method
         * {@link WritableGrid#getSampleType() getSampleType} is used to determine the type and
         * the bytelength (4 bytes for Float/Int and 8 bytes for doubleValues);
         * @param grid
         * @param output
         * @throws XuluGridFileException
         */
        public XuluGridFile(WritableGrid grid, File output)
                        throws XuluGridFileException {
                this.outputFile = output;
                //make a new MetaData object out of the grid
                RasterMetaData metaData = new RasterMetaData(grid);

                setByteLength(metaData);

                //write the metadata file
                writeWorldFileForMetaData(output, metaData);

                // Write out the GridFile (32Bit and float only until now)
                // for this go through the Grid line by line upwards

                try {
                        output.createNewFile();
                        gridFile = new BufferedRandomAccessFile(output.getAbsolutePath(),
                                        "rw", buffersize);

                    for (int y = metaData.getMinY(); y < metaData.getHeight()
                                        - metaData.getMinY(); y++)
                                for (int x = metaData.getMinY(); x < metaData.getWidth()
                                                - metaData.getMinX(); x++)
                                        gridFile.writeFloat(grid.getRasterSampleAsFloat(x, y));

                    gridFile.close();

                } catch (FileNotFoundException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                } catch (IOException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }
        }

        /**
         * Returns a partition of a {@link WritableGrid} as a
         * {@link WritableGridArray}. Notice, that the minX and minY values are
         * ignored and will be set to 0 (see {@link WritableGrid} for more
         * information about minX and minY). If the given {@link WritableGrid} is
         * not an instance of {@link WritableGridPartition} it is assumed that the
         * topLeft corner of the source grid is (0,0). If it is a
         * {@link WritableGridPartition} it is assumed that the the bounds of the
         * two Partition refer to the same coordinate system. Real coordinates are
         * supposed to reference the South(!)-West Corner.
         *
         * @param partitionBounds
         *            the rectangle describing the partition
         *
         * @return the new partition including the given corners
         * @throws XuluGridFileException
         *             if the reading goes wrong
         */
        public WritableGridPartition getPartitialGrid2D(
                        Rectangle partitionBounds) throws XuluGridFileException {
                synchronized (monitor){

                // get Metadata of the big Grid
                RasterMetaData bigMeta = this.metaData;

                // create MetaData for the new partitial Grid
                // Height an width of the partitial Grid
                int pHeight = (int) partitionBounds.getHeight();
                int pWidth = (int) partitionBounds.getWidth();

                // real coordinates of the partitial Grid (which are referenced by the
                // SW Corner)
                double pX = partitionBounds.getX() * bigMeta.getCellWidth()
                                + bigMeta.getX();
                double pY = (partitionBounds.getY() + partitionBounds.getHeight() - 1)
                                * bigMeta.getCellHeight() + bigMeta.getY();
                double pRealWidth = pWidth * bigMeta.getCellWidth();
                double pRealHeight = pHeight * bigMeta.getCellHeight();

                // the MetaData of the new Grid
                RasterMetaData pMetaData = new RasterMetaData(bigMeta.getDataType(),
                                pWidth, pHeight, 0, 0, pX, pY, pRealWidth, pRealHeight, bigMeta.getCoordinateReferenceSystem());

                // if the sourceGrid is a partition itself then we do not want to to
                // start reading from (0,0)
                // but shifted relative to the global coordinate system. So we calculate
                // now the start values:

                // for standard WritableGrids these are simply the bounds of the new
                // partition...
                int startX = (int) partitionBounds.getX();
                int startY = (int) partitionBounds.getY();

                try {
                        //seek to starting Position offset (= datavalues until startx)
                        long offset = ((startY) * metaData.getWidth() // number of rows to skip
                        + startX) // number of Colums to skip
                                        * dataValueSize; // length of one value in bytes
                        gridFile.seek(offset);
                        //the read-buffer is as wide as the grid (because we read line by line)
                        byte[] buffer = new byte[pWidth * dataValueSize];

                        // create empty return Grid
                        WritableGridPartition pGrid;

                        //switch structures are a bad smell, but necessary here because of the
                        //implementation of WritableGridArray
                        switch (bigMeta.getDataType()) {
                        case DataBuffer.TYPE_INT:
                                pGrid = new WritableGridArrayPartition.Integer(pMetaData,
                                                getRootID(), partitionBounds);

                                // read the data into the raster from left to right and upwards
                                for (int y = 0; y < pHeight; y++) {
                                        //for buffered reading read first into a bytearray
                                        gridFile.read(buffer);
                                        ByteArrayInputStream stream = new ByteArrayInputStream(
                                                        buffer);
                                        for (int x = 0; x < pWidth; x++) {
                                                int data = BufferedHelper.readIntFromStream(stream);
                                                pGrid.setRasterSample(data, x, y);
                                        }
                                        //seek to the next input position (if not last row)
                                        int seek = (metaData.getWidth() - pWidth) * dataValueSize;
                                        if (!(y == pHeight - 1))
                                                gridFile.skipBytes(seek);
                                }
                                break;

                        case DataBuffer.TYPE_DOUBLE:
                                pGrid = new WritableGridArrayPartition.Double(pMetaData,
                                                getRootID(), partitionBounds);

                                // read the data into the raster from left to right and upwards
                                for (int y = 0; y < pHeight; y++) {
                                        //for buffered reading read first into a bytearray
                                        gridFile.read(buffer);
                                        ByteArrayInputStream stream = new ByteArrayInputStream(
                                                        buffer);

                                        for (int x = 0; x < pWidth; x++) {
                                                double data = Double.longBitsToDouble(BufferedHelper
                                                                .readLongFromStream(stream));
                                                pGrid.setRasterSample(data, x, y);
                                        }
                                        //seek to the next input position (if not last row)
                                        int seek = (metaData.getWidth() - pWidth) * dataValueSize;
                                        if (!(y == pHeight - 1))
                                                gridFile.skipBytes(seek);
                                }
                                break;

                        case DataBuffer.TYPE_FLOAT:
                                pGrid = new WritableGridArrayPartition.Float(pMetaData,
                                                getRootID(), partitionBounds);

                                // read the data into the raster from left to right and downwards
                                System.out.print("[getX]");
                                for (int y = 0; y < pHeight; y++) {
                                        //for buffered reading read first into a bytearray
                                        gridFile.read(buffer);
                                        ByteArrayInputStream stream = new ByteArrayInputStream(
                                                        buffer);

                                        for (int x = 0; x < pWidth; x++) {
                                                float data = Float.intBitsToFloat(BufferedHelper
                                                                .readIntFromStream(stream));
                                                pGrid.setRasterSample(data, x, y);
                                        }

                                        // seek to the next input position (if not last row)
                                        int seek = (metaData.getWidth() - pWidth) * dataValueSize;
                                        if (!(y == pHeight - 1)) {
                                                long currentOffset = gridFile.getFilePointer();
                                                if (currentOffset + seek > gridFile.length())
                                                        System.err.println("Error: offset was "
                                                                        + gridFile.getFilePointer()
                                                                        + " and seek was " + seek
                                                                        + " but file has only " + gridFile.length()
                                                                        + " bytes");
                                                gridFile.skipBytes(seek);
                                        }
                                }
                                break;
                        default:
                                throw new UnsupportedOperationException("No value found");
                        }
                        return pGrid;

                } catch (IOException e) {

                        throw new XuluGridFileException(
                                        "IOException while operating on XuluGridFile. Reason was: \n"
                                                        + e.getCause());
                }
                }
        }

        /**
         *  @return the hashcode of this object
         *  @see SplittableResource#getRootID()
         */
        public int getRootID() {
                return this.hashCode();
        }

        /**
         * Overwrites the data at the location specified by the {@link Rectangle} with
         * the given partition-data. It is assumed that the topLeft corner is (0,0).
         * The method is synchronized over a static variable to ensure that only
         * one XuluGridFile is read/write at a time. This reduces (hopefully) harddisk
         * seek times and increases the overall performance.
         *
         * @param gridPartition the grid to be inserted
         * @param partitionBounds the excact location in coordinates of the baseGrid
         * @throws XuluGridFileException if the writing goes wrong
         */
        public void setPartition(WritableGrid gridPartition,
                        Rectangle partitionBounds) throws XuluGridFileException {
                synchronized (monitor){
                //Check if partition and the rectangle fit
                if ((gridPartition.getWidth() != partitionBounds.getWidth())
                                || (gridPartition.getHeight() != partitionBounds.getHeight()))
                        throw new UnsupportedOperationException(
                                        "The partition width/height does not match the rectangle width/height");

                if (this.metaData.getDataType() != gridPartition.getSampleType())
                        throw new UnsupportedOperationException("Datatypes incompatible: "
                                        + this.metaData.getDataType() + "!="
                                        + gridPartition.getSampleType());
                //the start coodinates
                int startX = (int) partitionBounds.getX();
                int startY = (int) partitionBounds.getY();
                int pHeight = (int) partitionBounds.getHeight();
                int pWidth = (int) partitionBounds.getWidth();

                try {
                        //                      seek to starting Position offset = datavalues until startx
                        long offset = ((startY) * metaData.getWidth() // number of rows to skip
                        + startX) // number of Colums to skip
                                        * dataValueSize; // length of one value in bytes
                        gridFile.seek(offset);
                        //the buffer is exactly as wide as the grid
                        ByteArrayOutputStream bos = new ByteArrayOutputStream(dataValueSize
                                        * pWidth);

                        switch (this.metaData.getDataType()) {
                        case DataBuffer.TYPE_FLOAT:
                                System.out.print("[setX]");
                                for (int y = 0; y < pHeight; y++) {
                                        for (int x = 0; x < pWidth; x++) {
                                                float data = gridPartition.getRasterSampleAsFloat(x, y);
                                                BufferedHelper
                                                                .writeInt(Float.floatToIntBits(data), bos);
                                        }
                                        //write buffered bytes to file
                                        gridFile.write(bos.toByteArray());
                                        //clear the bos
                                        bos.reset();
                                        // seek to the next input position
                                        gridFile.skipBytes((metaData.getWidth() - pWidth)
                                                        * dataValueSize);
                                }
                                break;
                        case DataBuffer.TYPE_INT:
                                for (int y = 0; y < pHeight; y++) {
                                        for (int x = 0; x < pWidth; x++) {
                                                int data = gridPartition.getRasterSampleAsInt(x, y);
                                                this.gridFile.writeInt(data);
                                        }
                                        // seek to the next input position
                                        gridFile.skipBytes((metaData.getWidth() - pWidth)
                                                        * dataValueSize);
                                }
                                break;
                        case DataBuffer.TYPE_DOUBLE:
                                for (int y = 0; y < pHeight; y++) {
                                        for (int x = 0; x < pWidth; x++) {
                                                double data = gridPartition.getRasterSampleAsDouble(x,
                                                                y);
                                                this.gridFile.writeDouble(data);
                                        }
                                        // seek to the next input position
                                        gridFile.skipBytes((metaData.getWidth() - pWidth)
                                                        * dataValueSize);
                                }
                        }

                } catch (Exception e) {
                        throw new XuluGridFileException(
                                        "Error while writing in XuluGridFile");
                }
                }
        }

        /**
         * Closes all used Filehandles. Should be called before destroying the
         * object.
         */
        public void close() {
                try {
                        gridFile.close();
                } catch (IOException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }
        }


        /**
         * @return the metadata for this object
         */
        public RasterMetaData getMetaData() {
                return metaData;
        }

        /**
         * The Xulu world file ist just a serialized RasterMetaData-Object!
         *
         * @param worldFile the world file
         * @return a {@link RasterMetaData} Object, that contains all the raster meta data
         * @throws XuluGridFileException if an error occures
         */
        public RasterMetaData readXuluGridWorldFile(File worldFile)
                        throws XuluGridFileException {
                try {
                        FileInputStream fis = new FileInputStream(worldFile);
                        ObjectInputStream ois = new ObjectInputStream(fis);
                        RasterMetaData meta = (RasterMetaData) ois.readObject();
                        ois.close();
                        fis.close();
                        return meta;

                } catch (IOException e) {
                        throw new XuluGridFileException(
                                        "IOException while reading Worldfile"
                                                        + worldFile.toString());
                } catch (ClassNotFoundException e) {
                        throw new XuluGridFileException(
                                        "ClassNotFoundException while reading Worldfile"
                                                        + worldFile.toString());
                }
        }

        /**
         * Writes the given Grid to a XuluGridFile. Warning: unbuffered write.
         *
         * @param grid the input grid
         * @param output the output file
         * @throws XuluGridFileException if something goes wrong
         */
        public static void writeToXuluGridFile(WritableGrid grid, File output)
                        throws XuluGridFileException {
                RasterMetaData metaData = new RasterMetaData(grid);

                writeWorldFileForMetaData(output, metaData);

                // Write out the GridFile (32Bit and float only until now)
                // for this go through the Grid line by line upwards

                try {
                        output.createNewFile();
                        RandomAccessFile randomFile = new RandomAccessFile(output, "rw");
                        //              fos = new FileOutputStream(output);
                        //            BufferedOutputStream bos = new BufferedOutputStream(fos);
                        //            ObjectOutputStream oos = new ObjectOutputStream(bos);
                        for (int y = metaData.getMinY(); y < metaData.getHeight()
                                        - metaData.getMinY(); y++)
                                for (int x = metaData.getMinY(); x < metaData.getWidth()
                                                - metaData.getMinX(); x++)
                                        randomFile.writeFloat(grid.getRasterSampleAsFloat(x, y));
                        //            oos.close();
                        //            bos.close();
                        //            fos.close();
                        randomFile.close();

                } catch (FileNotFoundException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                } catch (IOException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }
        }

        /**
         * Writes data out of a XuluGridFile into a {@link WritableGrid}. Warning: unbuffered. May be slow.
         * For buffered read use {@link #getWholeGrid()} instead.
         * @param grid the Grid in which the data should be written
         * @param input the File with the XuluGridFile
         *
         * */
        public static void readIntoGridFromXuluGridFile(WritableGrid grid,
                        File input) throws XuluGridFileException {
                synchronized (monitor) {

                RasterMetaData metaData = new RasterMetaData(grid);

                // Read from File into the given grid

                try {
                        RandomAccessFile randomFile = new RandomAccessFile(input, "r");
                        for (int y = metaData.getMinY(); y < metaData.getHeight()
                                        - metaData.getMinY(); y++)
                                for (int x = metaData.getMinY(); x < metaData.getWidth()
                                                - metaData.getMinX(); x++)
                                        grid.setRasterSample(randomFile.readFloat(), x, y);

                } catch (FileNotFoundException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                } catch (IOException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }
                }
        }

        /**
         * This File simply serializes the given {@link appl.util.RasterMetaData}
         * Object to the given file.
         */
        public static void writeWorldFileForMetaData(File XuluGridFile,
                        RasterMetaData meta) throws XuluGridFileException {
                try {
                        File xuluWorldFile = getWorldFileNameForGridFile(XuluGridFile);
                        xuluWorldFile.createNewFile();
                        FileOutputStream fos = new FileOutputStream(xuluWorldFile);
                        ObjectOutputStream oos = new ObjectOutputStream(fos);
                        oos.writeObject(meta);
                        oos.close();
                        fos.close();
                } catch (FileNotFoundException e) {
                        throw new XuluGridFileException(
                                        "Exception while processing WorldFile of XuluGridFile "
                                                        + XuluGridFile);
                } catch (IOException e) {
                        throw new XuluGridFileException(
                                        "Exception while processing WorldFile of XuluGridFile "
                                                        + XuluGridFile);
                }
        }


        /**
         * Returns a worldfilename. For this the extension xgrid is replaced by xworld
         */
        private static File getWorldFileNameForGridFile(File xuluGridFile)
                        throws XuluGridFileException {
                checkFileName(xuluGridFile);
                // create worldfilename by exchanging the file extension
                File worldFile = new File(xuluGridFile.getAbsolutePath().replaceAll(
                                ".xgrid$", ".xworld"));
                return worldFile;
        }

        /**
         * Checks if the given filename is a valid grid file name. To be valid the name must end with .xgrid
         * @param xuluGridFile the file to be checked
         * @throws XuluGridFileException if this is not the case
         *
         */
        static void checkFileName(File xuluGridFile) throws XuluGridFileException {
                if (!xuluGridFile.getAbsolutePath().endsWith(".xgrid"))
                        throw new XuluGridFileException(
                                        "XuluGridFiles must have the extension \".xgrid\"");

        }

        /**
         * gives back the whole grid as a {@link WritableGridArray} in memory! Notice
         * that this destroys all memory advantages of the GridFile. Is used e.g. by visualisation classes
         *
         */
        public WritableGrid getWholeGrid() throws XuluGridFileException {
                return getPartitialGrid2D(new Rectangle(0, 0, metaData.getWidth(),
                                metaData.getHeight()));
        }

        /**
         * @return the underlying file
         */
        public File getOutputFile() {
                return outputFile;
        }

}