parallel/test/SPMDTest_MultiGrid.java

package appl.parallel.test;

import java.awt.Rectangle;
import java.io.File;
import java.io.FileInputStream;
import java.io.Serializable;
import java.util.Random;
import java.util.Vector;

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;

import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import edu.bonn.xulu.io.ImportFactory;
import edu.bonn.xulu.plugin.io.grid.array.WritableGridArrayFactory;

import schmitzm.data.WritableGrid;
import schmitzm.data.WritableGridArray;
import schmitzm.data.WritableGridRaster;
import schmitzm.geotools.io.GeoImportUtil;

import appl.parallel.ComputingResourceContainer;
import appl.parallel.client.RemoteEventHandler;
import appl.parallel.client.ClientDataServer;
import appl.parallel.data.splittable.SplittableLLProxyGrid;
import appl.parallel.services.GlobalDiscoveryService;
import appl.parallel.spmd.AbstractSPMDTask;
import appl.parallel.spmd.SPMDClientController;
import appl.parallel.spmd.SPMDTask;
import appl.parallel.spmd.SPMDServerController;
import appl.parallel.spmd.split.SplittableGrid;
import appl.util.RasterMetaData;
import appl.util.RasterUtil;

/**
 * This is a minor modification of {@link SPMDTest}. It is only used to test the
 * Multi-Splittable functionality of the parallelization. For the test of all
 * other features and more detailed information see {@link SPMDTest}.
 * 
 * @author Dominik Appl
 */
public class SPMDTest_MultiGrid extends TestCase implements Serializable {

        transient private final Logger LOG = LogManager.getLogger(this.getClass()
                        .getName());

        transient private ClientDataServer client;
        transient private Vector<ComputingResourceContainer> computingResources;
        transient private SPMDClientController clientController;

        transient SplittableGrid gridInput;
        transient SplittableGrid remoteInput;
        transient SplittableGrid localOutput;
        transient SplittableGrid remoteOutput;

        /**
         * Constructor for SPMDTest.
         * 
         * @param name
         */
        public SPMDTest_MultiGrid(String name) {
                super(name);
        }

        /**
         * Returns the JUnit test suite that implements the <b>SPMDTest</b>
         * definition.
         */
        public static Test suite() {
                TestSuite sPMDTest = new TestSuite("SPMDTest");
//              sPMDTest.setArbiter(DefaultTestArbiter.INSTANCE).setId(
//                              "E873D66A70977ED2A98AAC10B54511DB");
//
//              sPMDTest.addTest(new SPMDTest("testSPMD").setId(
//                              "E873D66A70977ED2E5E8ED20B54511DB").setTestInvocationId(
//                              "E873D66A70977ED2EF7E3C00B54511DB"));
                return sPMDTest;
        }

        /**
         * @see junit.framework.TestCase#setUp()
         */
        protected void setUp() throws Exception {
                PropertyConfigurator.configure("xululog4j.cfg");
                computingResources = new GlobalDiscoveryService()
                                .getRemoteResources();

                for (ComputingResourceContainer computingRes : computingResources) {
                        System.out.println("DiscoveredRessource " + computingRes);
                }
                LOG.info("Discovery finished...");
                RemoteEventHandler handler = new RemoteEventHandler();
                client = new ClientDataServer(handler);
                clientController = new SPMDClientController(computingResources,null, client,handler);
                // ////////// Initialise data \\\\\\\\\\\\\
                WritableGridRaster baseGrid = GeoImportUtil
                                .readGridRasterFromArcInfoASCII(new File(
                                                "../Xulu-Data/minigrid2.arc")); // small data
//               "../Xulu-Data/large_clue_data/sc1gr0.0")); //large data
                gridInput = new SplittableLLProxyGrid(new WritableGridArrayFactory(),
                                new RasterMetaData(baseGrid));
                localOutput = new SplittableLLProxyGrid(new WritableGridArrayFactory(),
                                new RasterMetaData(baseGrid));
                remoteInput = new SplittableLLProxyGrid(new WritableGridArrayFactory(),
                                new RasterMetaData(baseGrid));
                remoteOutput = new SplittableLLProxyGrid(
                                new WritableGridArrayFactory(), new RasterMetaData(baseGrid));
                RasterUtil.copyInto(baseGrid, gridInput);
                RasterUtil.copyInto(baseGrid, remoteInput);
                //RasterUtil.copyInto(baseGrid, localOutput);
                //RasterUtil.copyInto(baseGrid, remoteOutput);
        }

        /**
         * @see junit.framework.TestCase#tearDown()
         */
        protected void tearDown() throws Exception {
                client.close();
                clientController.close();
                // GlobalDiscoveryService.getInstance().stopServices();
        }

        /**
         * TestSPMD
         * 
         * @throws Exception
         */
        public void testSPMD() throws Exception {
                int steps = 4;
                int neighborhoodRange = 3;
                System.out.println("Starting step 1: ");
                                
                /***********************************************************************
                 * START LOCAL ALGORITHM ******************** The simple Algorithm will
                 * calculate the average of the neighborhood und write the output to the
                 * coresponing cell in the output cell
                 **********************************************************************/

                Rectangle partition = new Rectangle(0, 0, gridInput.getWidth(),
                                gridInput.getHeight());
                if (gridInput.getWidth() < 20)
                        RasterUtil.printGrid(gridInput, 5, 1, "Input");
                
                for (int step = 0; step < steps; step++) {
//                       the overallsum is simply the sum over all elements
                        float overallSum = 0f;
                        long localStartTime = System.currentTimeMillis();
                        for (int y = 0; y < gridInput.getHeight(); y++)
                                for (int x = 0; x < gridInput.getWidth(); x++) {
                                        float tmp = gridInput.getRasterSampleAsFloat(x, y);
                                        if (!Float.isNaN(tmp))
                                                overallSum += tmp;
                                        // the local sum is simply the sum over all elements in the
                                        // neighborhood
                                        float localSum = 0;
                                        // number of cells over which the sum is calculated
                                        int noOfCells = 0;
                                        // for each cell: calculate the sum of all neighbors
                                        for (int y2 = y - neighborhoodRange; y2 <= y
                                                        + neighborhoodRange; y2++)
                                                for (int x2 = x - neighborhoodRange; x2 <= x
                                                                + neighborhoodRange; x2++)
                                                        // check if the coordinates are valid (inside the
                                                        // grid and not NaN)
                                                        if (partition.contains(x2, y2))
                                                                if (!Float.isNaN(gridInput
                                                                                .getRasterSampleAsFloat(x2, y2))) {
                                                                        localSum += gridInput
                                                                                        .getRasterSampleAsFloat(x2, y2);
                                                                        noOfCells++;
                                                                }

                                        localOutput.setRasterSample(localSum / (float) (noOfCells),
                                                        x, y);

                                }

                        System.out.print("Finished step " + (step+1) + " in "
                                        + ((System.currentTimeMillis() - localStartTime)) + " ms");
                        localStartTime = System.currentTimeMillis();
                        System.out.println(" with sum (local calculated): " + overallSum);
                        //copy values into results from output to input (for next step)
                        RasterUtil.copyInto(localOutput, gridInput);
                 }
                 if(gridInput.getWidth()<20)
                 RasterUtil.printGrid(localOutput, 5, 1, "Local Output");

                
                 
                 
                 /***********************************************************************
                 *                                                      START REMOTE ALGORITHM
                 *  ******************** All the same..now remotely********************/
                 
                 System.out
                                .println("******************* starting remote execution *****************");
                long remoteStartTime = System.currentTimeMillis();
                clientController.setNeighborhoodRange(neighborhoodRange);
//              initialize as MulitSplittable
                WritableGrid[] testSplittable = {remoteInput,remoteOutput};
                clientController.addToMultiDataSplitControl(testSplittable, "gridData");
                
//              clientController.addToSplitControl(remoteInput, "inputGrid");
//              clientController.addToSplitControl(remoteOutput, "outputGrid");
                System.out.println("After "
                                + ((System.currentTimeMillis() - remoteStartTime))
                                + "ms: finished with adding to splitcontroll");
                
                //********************** START STEP CONTROL *********************
                for (int step = 0; step < steps; step++) {
                        if(step > 0){
                                long updateStartTime = System.currentTimeMillis();
                                clientController.updateNeighborhood(remoteInput);
                                System.out.println("Update took:" + (System.currentTimeMillis()-updateStartTime));
                        }
                        Object[] sum;
                        try {
                                sum = clientController.runSPMDModelTask(
                                                new AverageNeighborhoodTestTask_MultiGrid(), neighborhoodRange);
                                //add results
                                float overallSum = 0;
                                for (int i = 0; i < sum.length; i++)
                                        overallSum += (Float) sum[i];
                                System.out.println("finished step "
                                                + (step+1) + " in "
                                                + ((System.currentTimeMillis() - remoteStartTime))
                                                + "ms " + " with sum "+
                                                + overallSum);
                                remoteStartTime = System.currentTimeMillis();
                                
                        } catch (Throwable e) {
                                // TODO Auto-generated catch block
                                e.printStackTrace();
                        }
                }
                //***************** END STEP CONTROL *****************************
                
                //After execution:
                // merge partitions and check if the local and remotely calculated
                // versions are equal
                remoteStartTime = System.currentTimeMillis();
                clientController.mergePartition(remoteOutput);
                System.out.println("Merging took another "
                                + ((System.currentTimeMillis() - remoteStartTime))
                                + "ms");
                // Thread.currentThread().sleep(10000);
                 if(remoteInput.getWidth()<20)
                         RasterUtil.printGrid(remoteOutput, 5, 1, "Remote Output");
                assertTrue(RasterUtil.checkEqual(remoteOutput, localOutput, true));
        }
}
1	mojays	2	package appl.parallel.test;
2
3			import java.awt.Rectangle;
4			import java.io.File;
5			import java.io.FileInputStream;
6			import java.io.Serializable;
7			import java.util.Random;
8			import java.util.Vector;
9
10			import junit.framework.Test;
11			import junit.framework.TestCase;
12			import junit.framework.TestSuite;
13
14			import org.apache.log4j.BasicConfigurator;
15			import org.apache.log4j.ConsoleAppender;
16			import org.apache.log4j.Layout;
17			import org.apache.log4j.Level;
18			import org.apache.log4j.LogManager;
19			import org.apache.log4j.Logger;
20			import org.apache.log4j.PropertyConfigurator;
21
22			import edu.bonn.xulu.io.ImportFactory;
23			import edu.bonn.xulu.plugin.io.grid.array.WritableGridArrayFactory;
24
25			import schmitzm.data.WritableGrid;
26			import schmitzm.data.WritableGridArray;
27			import schmitzm.data.WritableGridRaster;
28			import schmitzm.geotools.io.GeoImportUtil;
29
30			import appl.parallel.ComputingResourceContainer;
31			import appl.parallel.client.RemoteEventHandler;
32			import appl.parallel.client.ClientDataServer;
33			import appl.parallel.data.splittable.SplittableLLProxyGrid;
34			import appl.parallel.services.GlobalDiscoveryService;
35			import appl.parallel.spmd.AbstractSPMDTask;
36			import appl.parallel.spmd.SPMDClientController;
37			import appl.parallel.spmd.SPMDTask;
38			import appl.parallel.spmd.SPMDServerController;
39			import appl.parallel.spmd.split.SplittableGrid;
40			import appl.util.RasterMetaData;
41			import appl.util.RasterUtil;
42
43			/**
44			* This is a minor modification of {@link SPMDTest}. It is only used to test the
45			* Multi-Splittable functionality of the parallelization. For the test of all
46			* other features and more detailed information see {@link SPMDTest}.
47			*
48			* @author Dominik Appl
49			*/
50			public class SPMDTest_MultiGrid extends TestCase implements Serializable {
51
52			transient private final Logger LOG = LogManager.getLogger(this.getClass()
53			.getName());
54
55			transient private ClientDataServer client;
56			transient private Vector<ComputingResourceContainer> computingResources;
57			transient private SPMDClientController clientController;
58
59			transient SplittableGrid gridInput;
60			transient SplittableGrid remoteInput;
61			transient SplittableGrid localOutput;
62			transient SplittableGrid remoteOutput;
63
64			/**
65			* Constructor for SPMDTest.
66			*
67			* @param name
68			*/
69			public SPMDTest_MultiGrid(String name) {
70			super(name);
71			}
72
73			/**
74			* Returns the JUnit test suite that implements the <b>SPMDTest</b>
75			* definition.
76			*/
77			public static Test suite() {
78			TestSuite sPMDTest = new TestSuite("SPMDTest");
79			// sPMDTest.setArbiter(DefaultTestArbiter.INSTANCE).setId(
80			// "E873D66A70977ED2A98AAC10B54511DB");
81			//
82			// sPMDTest.addTest(new SPMDTest("testSPMD").setId(
83			// "E873D66A70977ED2E5E8ED20B54511DB").setTestInvocationId(
84			// "E873D66A70977ED2EF7E3C00B54511DB"));
85			return sPMDTest;
86			}
87
88			/**
89			* @see junit.framework.TestCase#setUp()
90			*/
91			protected void setUp() throws Exception {
92			PropertyConfigurator.configure("xululog4j.cfg");
93			computingResources = new GlobalDiscoveryService()
94			.getRemoteResources();
95
96			for (ComputingResourceContainer computingRes : computingResources) {
97			System.out.println("DiscoveredRessource " + computingRes);
98			}
99			LOG.info("Discovery finished...");
100			RemoteEventHandler handler = new RemoteEventHandler();
101			client = new ClientDataServer(handler);
102			clientController = new SPMDClientController(computingResources,null, client,handler);
103			// ////////// Initialise data \\\\\\\\\\\\\
104			WritableGridRaster baseGrid = GeoImportUtil
105			.readGridRasterFromArcInfoASCII(new File(
106			"../Xulu-Data/minigrid2.arc")); // small data
107			// "../Xulu-Data/large_clue_data/sc1gr0.0")); //large data
108			gridInput = new SplittableLLProxyGrid(new WritableGridArrayFactory(),
109			new RasterMetaData(baseGrid));
110			localOutput = new SplittableLLProxyGrid(new WritableGridArrayFactory(),
111			new RasterMetaData(baseGrid));
112			remoteInput = new SplittableLLProxyGrid(new WritableGridArrayFactory(),
113			new RasterMetaData(baseGrid));
114			remoteOutput = new SplittableLLProxyGrid(
115			new WritableGridArrayFactory(), new RasterMetaData(baseGrid));
116			RasterUtil.copyInto(baseGrid, gridInput);
117			RasterUtil.copyInto(baseGrid, remoteInput);
118			//RasterUtil.copyInto(baseGrid, localOutput);
119			//RasterUtil.copyInto(baseGrid, remoteOutput);
120			}
121
122			/**
123			* @see junit.framework.TestCase#tearDown()
124			*/
125			protected void tearDown() throws Exception {
126			client.close();
127			clientController.close();
128			// GlobalDiscoveryService.getInstance().stopServices();
129			}
130
131			/**
132			* TestSPMD
133			*
134			* @throws Exception
135			*/
136			public void testSPMD() throws Exception {
137			int steps = 4;
138			int neighborhoodRange = 3;
139			System.out.println("Starting step 1: ");
140
141			/***********************************************************************
142			* START LOCAL ALGORITHM ******************** The simple Algorithm will
143			* calculate the average of the neighborhood und write the output to the
144			* coresponing cell in the output cell
145			**********************************************************************/
146
147			Rectangle partition = new Rectangle(0, 0, gridInput.getWidth(),
148			gridInput.getHeight());
149			if (gridInput.getWidth() < 20)
150			RasterUtil.printGrid(gridInput, 5, 1, "Input");
151
152			for (int step = 0; step < steps; step++) {
153			// the overallsum is simply the sum over all elements
154			float overallSum = 0f;
155			long localStartTime = System.currentTimeMillis();
156			for (int y = 0; y < gridInput.getHeight(); y++)
157			for (int x = 0; x < gridInput.getWidth(); x++) {
158			float tmp = gridInput.getRasterSampleAsFloat(x, y);
159			if (!Float.isNaN(tmp))
160			overallSum += tmp;
161			// the local sum is simply the sum over all elements in the
162			// neighborhood
163			float localSum = 0;
164			// number of cells over which the sum is calculated
165			int noOfCells = 0;
166			// for each cell: calculate the sum of all neighbors
167			for (int y2 = y - neighborhoodRange; y2 <= y
168			+ neighborhoodRange; y2++)
169			for (int x2 = x - neighborhoodRange; x2 <= x
170			+ neighborhoodRange; x2++)
171			// check if the coordinates are valid (inside the
172			// grid and not NaN)
173			if (partition.contains(x2, y2))
174			if (!Float.isNaN(gridInput
175			.getRasterSampleAsFloat(x2, y2))) {
176			localSum += gridInput
177			.getRasterSampleAsFloat(x2, y2);
178			noOfCells++;
179			}
180
181			localOutput.setRasterSample(localSum / (float) (noOfCells),
182			x, y);
183
184			}
185
186			System.out.print("Finished step " + (step+1) + " in "
187			+ ((System.currentTimeMillis() - localStartTime)) + " ms");
188			localStartTime = System.currentTimeMillis();
189			System.out.println(" with sum (local calculated): " + overallSum);
190			//copy values into results from output to input (for next step)
191			RasterUtil.copyInto(localOutput, gridInput);
192			}
193			if(gridInput.getWidth()<20)
194			RasterUtil.printGrid(localOutput, 5, 1, "Local Output");
195
196
197
198
199			/***********************************************************************
200			* START REMOTE ALGORITHM
201			* ****************** All the same..now remotely******************/
202
203			System.out
204			.println("***************** starting remote execution ***************");
205			long remoteStartTime = System.currentTimeMillis();
206			clientController.setNeighborhoodRange(neighborhoodRange);
207			// initialize as MulitSplittable
208			WritableGrid[] testSplittable = {remoteInput,remoteOutput};
209			clientController.addToMultiDataSplitControl(testSplittable, "gridData");
210
211			// clientController.addToSplitControl(remoteInput, "inputGrid");
212			// clientController.addToSplitControl(remoteOutput, "outputGrid");
213			System.out.println("After "
214			+ ((System.currentTimeMillis() - remoteStartTime))
215			+ "ms: finished with adding to splitcontroll");
216
217			//******************** START STEP CONTROL *******************
218			for (int step = 0; step < steps; step++) {
219			if(step > 0){
220			long updateStartTime = System.currentTimeMillis();
221			clientController.updateNeighborhood(remoteInput);
222			System.out.println("Update took:" + (System.currentTimeMillis()-updateStartTime));
223			}
224			Object[] sum;
225			try {
226			sum = clientController.runSPMDModelTask(
227			new AverageNeighborhoodTestTask_MultiGrid(), neighborhoodRange);
228			//add results
229			float overallSum = 0;
230			for (int i = 0; i < sum.length; i++)
231			overallSum += (Float) sum[i];
232			System.out.println("finished step "
233			+ (step+1) + " in "
234			+ ((System.currentTimeMillis() - remoteStartTime))
235			+ "ms " + " with sum "+
236			+ overallSum);
237			remoteStartTime = System.currentTimeMillis();
238
239			} catch (Throwable e) {
240			// TODO Auto-generated catch block
241			e.printStackTrace();
242			}
243			}
244			//*************** END STEP CONTROL ***************************
245
246			//After execution:
247			// merge partitions and check if the local and remotely calculated
248			// versions are equal
249			remoteStartTime = System.currentTimeMillis();
250			clientController.mergePartition(remoteOutput);
251			System.out.println("Merging took another "
252			+ ((System.currentTimeMillis() - remoteStartTime))
253			+ "ms");
254			// Thread.currentThread().sleep(10000);
255			if(remoteInput.getWidth()<20)
256			RasterUtil.printGrid(remoteOutput, 5, 1, "Remote Output");
257			assertTrue(RasterUtil.checkEqual(remoteOutput, localOutput, true));
258			}
259			}