Context Navigation

DCWormsStatistics.java @ 1597

Revision 1597, 78.4 KB checked in by wojtekp, 8 years ago (diff)

Property svn:mime-type set to `text/plain`

Line
1	package simulator.stats.implementation;
2
3	import java.awt.Color;
4	import java.awt.Paint;
5	import java.io.File;
6	import java.io.FileOutputStream;
7	import java.io.IOException;
8	import java.io.PrintStream;
9	import java.text.DecimalFormat;
10	import java.text.NumberFormat;
11	import java.util.ArrayList;
12	import java.util.Calendar;
13	import java.util.Collections;
14	import java.util.Comparator;
15	import java.util.Date;
16	import java.util.HashMap;
17	import java.util.HashSet;
18	import java.util.LinkedList;
19	import java.util.List;
20	import java.util.Map;
21	import java.util.Set;
22	import java.util.SortedMap;
23	import java.util.TreeMap;
24
25	import org.apache.commons.lang.ArrayUtils;
26	import org.apache.commons.logging.Log;
27	import org.apache.commons.logging.LogFactory;
28	import org.jfree.chart.ChartFactory;
29	import org.jfree.chart.ChartRenderingInfo;
30	import org.jfree.chart.ChartUtilities;
31	import org.jfree.chart.JFreeChart;
32	import org.jfree.chart.axis.AxisLocation;
33	import org.jfree.chart.axis.DateAxis;
34	import org.jfree.chart.axis.NumberAxis;
35	import org.jfree.chart.encoders.ImageFormat;
36	import org.jfree.chart.labels.CategoryItemLabelGenerator;
37	import org.jfree.chart.labels.ItemLabelAnchor;
38	import org.jfree.chart.labels.ItemLabelPosition;
39	import org.jfree.chart.labels.StandardCategoryItemLabelGenerator;
40	import org.jfree.chart.plot.CategoryPlot;
41	import org.jfree.chart.plot.CombinedDomainXYPlot;
42	import org.jfree.chart.plot.PlotOrientation;
43	import org.jfree.chart.plot.XYPlot;
44	import org.jfree.chart.renderer.category.CategoryItemRenderer;
45	import org.jfree.chart.renderer.category.GanttRenderer;
46	import org.jfree.chart.renderer.xy.XYStepAreaRenderer;
47	import org.jfree.chart.title.TextTitle;
48	import org.jfree.chart.title.Title;
49	import org.jfree.data.gantt.TaskSeries;
50	import org.jfree.data.gantt.TaskSeriesCollection;
51	import org.jfree.data.time.FixedMillisecond;
52	import org.jfree.data.xy.XYDataset;
53	import org.jfree.data.xy.XYSeries;
54	import org.jfree.data.xy.XYSeriesCollection;
55	import org.jfree.ui.TextAnchor;
56	import org.joda.time.DateTime;
57	import org.joda.time.DateTimeUtilsExt;
58
59	import schedframe.ExecutablesList;
60	import schedframe.SimulatedEnvironment;
61	import schedframe.resources.CustomResourceType;
62	import schedframe.resources.ResourceType;
63	import schedframe.resources.computing.ComputingResource;
64	import schedframe.resources.computing.extensions.Extension;
65	import schedframe.resources.computing.extensions.ExtensionList;
66	import schedframe.resources.computing.extensions.ExtensionType;
67	import schedframe.resources.computing.profiles.energy.EnergyExtension;
68	import schedframe.resources.computing.profiles.energy.MeasurementHistory;
69	import schedframe.resources.computing.profiles.energy.airthroughput.AirflowValue;
70	import schedframe.resources.computing.profiles.energy.power.PowerUsage;
71	import schedframe.resources.computing.profiles.energy.thermal.TemperatureValue;
72	import schedframe.resources.computing.profiles.load.LoadExtension;
73	import schedframe.resources.devices.Device;
74	import schedframe.resources.devices.PhysicalResource;
75	import schedframe.resources.units.ProcessingElements;
76	import schedframe.resources.units.ResourceUnit;
77	import schedframe.resources.units.ResourceUnitName;
78	import schedframe.resources.units.StandardResourceUnitName;
79	import schedframe.scheduling.ExecutionHistoryItem;
80	import schedframe.scheduling.ResourceItem;
81	import schedframe.scheduling.Scheduler;
82	import schedframe.scheduling.manager.tasks.JobRegistry;
83	import schedframe.scheduling.manager.tasks.JobRegistryImpl;
84	import schedframe.scheduling.tasks.Job;
85	import schedframe.scheduling.tasks.JobInterface;
86	import simulator.ConfigurationOptions;
87	import simulator.DCWormsConstants;
88	import simulator.DataCenterWorkloadSimulator;
89	import simulator.GenericUser;
90	import simulator.stats.CoolEmAllMetricsCalculator;
91	import simulator.stats.DCwormsAccumulator;
92	import simulator.stats.DCwormsMetricsCalculator;
93	import simulator.stats.MetricsCalculator;
94	import simulator.stats.SimulationStatistics;
95	import simulator.stats.implementation.out.CoolEmAllStringSerializer;
96	import simulator.stats.implementation.out.StringSerializer;
97	import csiro.mit.utils.jfreechart.timetablechart.TimetableChartFactory;
98	import csiro.mit.utils.jfreechart.timetablechart.data.Timetable;
99	import csiro.mit.utils.jfreechart.timetablechart.data.TimetableEventGroup;
100	import csiro.mit.utils.jfreechart.timetablechart.data.TimetableEventSource;
101	import csiro.mit.utils.jfreechart.timetablechart.renderer.TimetableRenderer;
102	import dcworms.schedframe.scheduling.ExecTask;
103	import dcworms.schedframe.scheduling.Executable;
104	import eduni.simjava.Sim_stat;
105	import example.energy.coolemall.CoolEmAllTestbedMeasurements;
106	import gridsim.dcworms.DCWormsTags;
107
108	public class DCWormsStatistics implements SimulationStatistics {
109
110	private Log log = LogFactory.getLog(DCWormsStatistics.class);
111
112	protected static float ALPHA = 0.5f;
113	protected static int GANTT_WIDTH = 1200;
114
115	protected static final int MILLI_SEC = 1000;
116
117	protected static final String TASKS_STATISTICS_OUTPUT_FILE_NAME = "Tasks.txt";
118	protected static final String JOBS_STATISTICS_OUTPUT_FILE_NAME = "Jobs.txt";
119
120	protected static final String SIMULATION_STATISTICS_OUTPUT_FILE_NAME = "Simulation.txt";
121	protected static final String RESOURCEOCCUPANCY_STATISTICS_OUTPUT_FILE_NAME = "ResourceOccupancy.txt";
122	protected static final String RESOURCEUTILIZATION_STATISTICS_OUTPUT_FILE_NAME = "ResourceUtilization.txt";
123	protected static final String ENERGYUSAGE_STATISTICS_OUTPUT_FILE_NAME = "EnergyUsage.txt";
124	protected static final String AIRFLOW_STATISTICS_OUTPUT_FILE_NAME = "Airflow.txt";
125	protected static final String TEMPERATURE_STATISTICS_OUTPUT_FILE_NAME = "Temperature.txt";
126	protected static final String USEFUL_WORK_STATISTICS_OUTPUT_FILE_NAME = "UsefulWork.txt";
127	protected static final String DEVICE_STATISTICS_OUTPUT_FILE_NAME = "Devices.txt";
128	protected static final String NODES_AVAILABILITY_STATISTICS_OUTPUT_FILE_NAME = "NodesAvailability.txt";
129	protected static final String METRICS_STATISTICS_OUTPUT_FILE_NAME = "Metrics.txt";
130	protected static final String RESOURCE_HISTORY_OUTPUT_FILE_NAME = "ResourceHistyory.txt";
131
132	protected static final String DIAGRAMS_FILE_NAME_PREFIX = "Chart_";
133	protected static final String STATS_FILE_NAME_PREFIX = "Stats_";
134
135	protected static final String SEPARATOR = ";";
136
137	protected String outputFolderName;
138
139	protected String simulationIdentifier;
140	protected ConfigurationOptions configuration;
141
142	protected GSSAccumulatorsStats accStats;
143	protected Map<String, DCwormsAccumulator> statsData;
144
145	protected GenericUser users;
146	protected SimulatedEnvironment resourceController;
147	protected boolean generateDiagrams = true;
148	protected StringSerializer serializer;
149	protected long startSimulationTime;
150	protected long endSimulationTime;
151
152	//RESOURCES
153	protected Map<ComputingResource, List<ResStat>> peStats;
154
155
156	protected Timetable ganttDiagramTimetable;
157	protected Map<String, List<XYDataset>> resourcePowerUsageDiagrams;
158	protected Map<String, List<XYDataset>> resourceAirflowDiagrams;
159	protected Map<String, List<XYDataset>> resourceTemperatureDiagrams;
160	protected Map<String, List<XYDataset>> resourceOccupancyDiagrams;
161	protected Map<String, List<XYDataset>> resourceUtilizationDiagrams;
162
163	//TASKS
164	protected int numOfdelayedTasks = 0;
165	protected int numOfNotExecutedTasks = 0;
166	protected double maxCj = 0;
167
168	protected boolean allTasksFinished;
169	protected TaskSeriesCollection ganttDiagramTaskSeriesCollection;
170	protected TaskSeriesCollection ganttDiagramWaitingTimeSeriesCollection;
171	protected Map<String, Set<ComputingResource>> taskToPEMap;
172
173	protected JobRegistry jobRegistry;
174	protected MetricsCalculator metCalc;
175
176	public DCWormsStatistics(String simulationIdentifier, ConfigurationOptions co, GenericUser users,
177	String outputFolderName, SimulatedEnvironment resourceController) {
178	this.simulationIdentifier = simulationIdentifier;
179	this.configuration = co;
180	this.users = users;
181	this.outputFolderName = outputFolderName;
182	this.resourceController = resourceController;
183	this.jobRegistry = new JobRegistryImpl();
184
185	if(users.isSimStartTimeDefined())
186	this.startSimulationTime = DateTimeUtilsExt.getOffsetTime().getTimeInMillis();
187	else
188	this.startSimulationTime = new DateTime(users.getSubmissionStartTime() * 1000l).getMillis();
189
190	this.endSimulationTime = DateTimeUtilsExt.currentTimeMillis();
191
192	init();
193	}
194
195	private void init() {
196	taskToPEMap = new HashMap<String, Set<ComputingResource>>(jobRegistry.getFinishedTasks().size());
197
198	statsData = new HashMap<String, DCwormsAccumulator>(2);
199	accStats = new GSSAccumulatorsStats();
200
201	this.serializer = new StringSerializer();
202	this.metCalc = new DCwormsMetricsCalculator(startSimulationTime, endSimulationTime, endSimulationTime);
203
204	if(DataCenterWorkloadSimulator.MODE.equals("CoolEmAll")){
205	this.serializer = new CoolEmAllStringSerializer();
206	this.metCalc = new CoolEmAllMetricsCalculator(startSimulationTime, endSimulationTime, endSimulationTime);
207	}
208
209	this.serializer.setDefaultNumberFormat(DataCenterWorkloadSimulator.DFAULT_NUMBER_FORMAT);
210	}
211
212	public void generateStatistics() {
213
214	long s = 0;
215	long e = 0;
216
217	log.info("gatherResourceStatistics");
218	s = System.currentTimeMillis();
219	gatherResourceStatistics();
220	e = System.currentTimeMillis();
221	log.info("time in sec: " + ((e - s) / MILLI_SEC));
222
223	log.info("gatherTaskStatistics");
224	s = System.currentTimeMillis();
225	gatherTaskStatistics();
226	e = System.currentTimeMillis();
227	log.info("time in sec: " + ((e - s) / MILLI_SEC));
228
229	log.info("saveSimulationStatistics");
230	s = System.currentTimeMillis();
231	saveSimulationStatistics();
232	e = System.currentTimeMillis();
233	log.info("time in sec: " + ((e - s) / MILLI_SEC));
234	}
235
236
237	public String getOutputFolderName() {
238	return outputFolderName;
239	}
240
241	public void saveSimulationStatistics() {
242	PrintStream simulationStatsFile = null;
243	if (configuration.createsimulationstatistics) {
244	try {
245	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX + simulationIdentifier + "_"
246	+ SIMULATION_STATISTICS_OUTPUT_FILE_NAME);
247	simulationStatsFile = new PrintStream(new FileOutputStream(file));
248	} catch (IOException e) {
249	e.printStackTrace();
250	}
251	}
252	if (simulationStatsFile != null) {
253	Object txt = accStats.serialize(this.serializer);
254
255	DecimalFormat df = new DecimalFormat();
256	df.setMaximumFractionDigits(3);
257	df.setGroupingUsed(false);
258
259	/*log.info("#STATS " + "Performance statistics");
260	log.info("#STATS " + "Makespan: " + accStats.makespan.getMean() + " [s]");
261	log.info("#STATS " + "Task completion time - " + "min: " + accStats.meanTaskCompletionTime.getMin() + "; max: " + accStats.meanTaskCompletionTime.getMax() + " [s]");
262	log.info("#STATS " + "Mean task execution time: " + accStats.meanTaskExecutionTime.getMean() + " [s]");
263	log.info("#STATS " + "System load: " + accStats.meanTotalLoad.getMean() * 100 + " [%]");*/
264
265	System.out.println("#STATS " + "===== Performance statistics =====");
266	System.out.println("#STATS " + "Makespan: " + df.format(accStats.makespan.getMean()) + " [s]");
267	System.out.println("#STATS " + "Task completion time - " + "min: " + df.format(accStats.meanTaskCompletionTime.getMin()) + " [s]" + "; max: " + df.format(accStats.meanTaskCompletionTime.getMax()) + " [s]" + "; mean: " + df.format(accStats.meanTaskCompletionTime.getMean()) + " [s]");
268	System.out.println("#STATS " + "Mean task execution time: " + df.format(accStats.meanTaskExecutionTime.getMean()) + " [s]");
269	System.out.println("#STATS " + "System occupancy: " + df.format(accStats.meanTotalOccupancy.getMean() * 100) + " [%]");
270	System.out.println("#STATS " + "System load: " + df.format(accStats.meanTotalLoad.getMean() * 100) + " [%]");
271
272	simulationStatsFile.println(txt);
273	}
274
275	if (simulationStatsFile != null) {
276	simulationStatsFile.close();
277	}
278	}
279
280	/*********** RESOURCE STATISTICS SECTION ************/
281	public void gatherResourceStatistics() {
282
283	HashMap<String, List<Stats>> type_stats = new HashMap<String, List<Stats>>();
284
285	for(String resourceTypeName: resourceController.getComputingResourceLayers()){
286	List<Stats> cStats = new ArrayList<Stats>();
287	/*if(ArrayUtils.contains(configuration.compResForUtilizationText, resourceTypeName))
288	cStats.add(Stats.textLoad);
289	if(ArrayUtils.contains(configuration.compResForEnergyText, resourceTypeName))
290	cStats.add(Stats.textEnergy);
291	if(ArrayUtils.contains(configuration.compResForAirflowText, resourceTypeName))
292	cStats.add(Stats.textAirFlow);
293	if(ArrayUtils.contains(configuration.compResForTemperatureText, resourceTypeName))
294	cStats.add(Stats.textTemperature);*/
295	cStats.add(Stats.textLoad);
296	if(ArrayUtils.contains(configuration.compResForUtilizationChart, resourceTypeName))
297	cStats.add(Stats.chartLoad);
298	cStats.add(Stats.textEnergy);
299	if(ArrayUtils.contains(configuration.compResForEnergyChart, resourceTypeName))
300	cStats.add(Stats.chartEnergy);
301	cStats.add(Stats.textAirflow);
302	if(ArrayUtils.contains(configuration.compResForAirflowChart, resourceTypeName))
303	cStats.add(Stats.chartAirflow);
304	cStats.add(Stats.textTemperature);
305	if(ArrayUtils.contains(configuration.compResForTemperatureChart, resourceTypeName))
306	cStats.add(Stats.chartTemperature);
307	type_stats.put(resourceTypeName, cStats);
308
309	}
310
311	resourceOccupancyDiagrams = new HashMap<String, List<XYDataset>>(2);
312	resourceUtilizationDiagrams = new HashMap<String, List<XYDataset>>(2);
313	resourcePowerUsageDiagrams = new HashMap<String, List<XYDataset>>(2);
314	resourceAirflowDiagrams = new HashMap<String, List<XYDataset>>(2);
315	resourceTemperatureDiagrams = new HashMap<String, List<XYDataset>>(2);
316
317	ganttDiagramTimetable = new Timetable(new FixedMillisecond(
318	startSimulationTime), new FixedMillisecond(endSimulationTime));
319
320
321	PrintStream resourceOccupancyStatsFile = null;
322	try {
323	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
324	+ simulationIdentifier + "_"
325	+ RESOURCEOCCUPANCY_STATISTICS_OUTPUT_FILE_NAME);
326	resourceOccupancyStatsFile = new PrintStream(new FileOutputStream(file));
327	} catch (IOException e) {
328	resourceOccupancyStatsFile = null;
329	}
330
331	PrintStream resourceUtilizationStatsFile = null;
332	try {
333	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
334	+ simulationIdentifier + "_"
335	+ RESOURCEUTILIZATION_STATISTICS_OUTPUT_FILE_NAME);
336	resourceUtilizationStatsFile = new PrintStream(new FileOutputStream(file));
337	} catch (IOException e) {
338	resourceUtilizationStatsFile = null;
339	}
340
341	PrintStream energyStatsFile = null;
342	try {
343	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
344	+ simulationIdentifier + "_"
345	+ ENERGYUSAGE_STATISTICS_OUTPUT_FILE_NAME);
346	energyStatsFile = new PrintStream(new FileOutputStream(file));
347	} catch (IOException e) {
348	energyStatsFile = null;
349	}
350
351	PrintStream airflowStatsFile = null;
352	try {
353	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
354	+ simulationIdentifier + "_"
355	+ AIRFLOW_STATISTICS_OUTPUT_FILE_NAME);
356	airflowStatsFile = new PrintStream(new FileOutputStream(file));
357	} catch (IOException e) {
358	airflowStatsFile = null;
359	}
360
361	PrintStream temperatureStatsFile = null;
362	try {
363	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
364	+ simulationIdentifier + "_"
365	+ TEMPERATURE_STATISTICS_OUTPUT_FILE_NAME);
366	temperatureStatsFile = new PrintStream(new FileOutputStream(file));
367	} catch (IOException e) {
368	temperatureStatsFile = null;
369	}
370
371	PrintStream deviceStatsFile = null;
372	try {
373	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
374	+ simulationIdentifier + "_"
375	+ DEVICE_STATISTICS_OUTPUT_FILE_NAME);
376	deviceStatsFile = new PrintStream(new FileOutputStream(file));
377	} catch (IOException e) {
378	deviceStatsFile = null;
379	}
380
381	/*PrintStream nodesAvailabilityStatsFile = null;
382	try {
383	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
384	+ simulationIdentifier + "_"
385	+ NODES_AVAILABILITY_STATISTICS_OUTPUT_FILE_NAME);
386	nodesAvailabilityStatsFile = new PrintStream(new FileOutputStream(file));
387	} catch (IOException e) {
388	nodesAvailabilityStatsFile = null;
389	}*/
390
391	PrintStream metricsStatsFile = null;
392	try {
393	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
394	+ simulationIdentifier + "_"
395	+ METRICS_STATISTICS_OUTPUT_FILE_NAME);
396	metricsStatsFile = new PrintStream(new FileOutputStream(file));
397	} catch (IOException e) {
398	metricsStatsFile = null;
399	}
400
401	PrintStream usefulWorkStatsFile = null;
402	try {
403	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX
404	+ simulationIdentifier + "_"
405	+ USEFUL_WORK_STATISTICS_OUTPUT_FILE_NAME);
406	usefulWorkStatsFile = new PrintStream(new FileOutputStream(file));
407	} catch (IOException e) {
408	usefulWorkStatsFile = null;
409	}
410
411	peStats = gatherPEStatsMulti(jobRegistry.getTasks());
412	peStatsPostProcessing(peStats);
413
414	if (configuration.creatediagrams_gantt) {
415	createGanttDiagram(peStats);
416	}
417
418	for(String resourceTypeName: resourceController.getComputingResourceLayers()){
419	DCwormsAccumulator resourceEnergyAccumulator = new DCwormsAccumulator();
420	DCwormsAccumulator maxResourceEnergyAccumulator = new DCwormsAccumulator();
421	DCwormsAccumulator calculationsEnergyAccumulator = new DCwormsAccumulator();
422	//DCwormsAccumulator tempAboveThresholdAccumulator = new DCwormsAccumulator();
423	//DCwormsAccumulator tempAboveThresholdAccumulatorH = new DCwormsAccumulator();
424	List<ComputingResource> compResources = new ArrayList<ComputingResource>();
425	for(ComputingResource compRes: resourceController.getComputingResources() ){
426	compResources.addAll(compRes.getDescendantsByType(new CustomResourceType(resourceTypeName)));
427	}
428	if(resourceController.getComputingResources().get(0).getType().getName().equals(resourceTypeName))
429	compResources.addAll(resourceController.getComputingResources());
430	if(type_stats.containsKey(resourceTypeName)){
431
432
433	for(ComputingResource compResource: compResources){
434	ResourceUsageStats resourceUtilization = null;
435	ResourceUsageStats resourceOccupancy = null;
436	ResourcePowerStats energyUsage = null;
437	ResourceAirflowStats airflow = null;
438	ResourceTemperatureStats temperature = null;
439	ResourceUsefulWorkStats usefulWork = null;
440
441	if(type_stats.get(resourceTypeName).contains(Stats.textLoad)){
442	resourceOccupancy = gatherResourceOccupancyStats(compResource, peStats);
443	resourceOccupancy.setMeanValue(calculateMeanValue(resourceOccupancy));
444	if (resourceOccupancyStatsFile != null) {
445	Object txt = resourceOccupancy.serialize(serializer);
446	resourceOccupancyStatsFile.print(txt);
447	}
448
449	resourceUtilization = gatherResourceUtlizationStats(compResource);
450	resourceUtilization.setMeanValue(calculateMeanValue(resourceUtilization));
451	if (resourceUtilizationStatsFile != null) {
452	Object txt = resourceUtilization.serialize(serializer);
453	resourceUtilizationStatsFile.print(txt);
454	}
455	}
456
457	if(type_stats.get(resourceTypeName).contains(Stats.chartLoad)){
458	if (configuration.creatediagrams_resutilization) {
459	createResourceOccupancyDiagram(resourceOccupancy);
460	createResourceUtilizationDiagram(resourceUtilization);
461	}
462	}
463
464	if(type_stats.get(resourceTypeName).contains(Stats.textEnergy)){
465	energyUsage = gatherResourcePowerConsumptionStats(compResource);
466	energyUsage.setMeanValue(calculateMeanValue(energyUsage));
467	energyUsage.setSumValue(energyUsage.getMeanValue() * (endSimulationTime - startSimulationTime) / (3600 * MILLI_SEC));
468	resourceEnergyAccumulator.add(energyUsage.getSumValue());
469	maxResourceEnergyAccumulator.add(calculateMaxValue(energyUsage));
470	double calculationsEnergy = 0;
471	try {
472	boolean doCalc = false;
473	double load = 0;
474	long prevTimestamp = 0;
475	for(Long key: energyUsage.getHistory().keySet()){
476	if(doCalc){
477	if(resourceTypeName.equals("Rack")){
478	calculationsEnergy = calculationsEnergy + CoolEmAllTestbedMeasurements.OTHER_DEVICES_POWER_CONSUMPTION * (key - prevTimestamp) /(3600.0 * MILLI_SEC);
479	}/* else if(resourceTypeName.equals("Processor")){
480	Processor proc =( Processor) compResource;
481	double maxPower = proc.getPowerInterface().getLowestPState().getLoadPowerUsage().get(100.0);
482	calculationsEnergy = calculationsEnergy + load * maxPower * (key - prevTimestamp) /(3600.0 * MILLI_SEC);
483	}*/
484	else
485	calculationsEnergy = calculationsEnergy + load * energyUsage.getHistory().get(prevTimestamp) * (key - prevTimestamp) /(3600.0 * MILLI_SEC);
486	doCalc = false;
487	load = 0;
488	}
489
490	if(resourceUtilization.getHistory().get(key) != null){
491	load = resourceUtilization.getHistory().get(key);
492	if(load > 0 ){
493	doCalc = true;
494	prevTimestamp = key;
495	}
496	}
497	}
498
499	calculationsEnergyAccumulator.add(calculationsEnergy);
500	} catch (Exception e) {
501	e.printStackTrace();
502	}
503	EnergyExtension een = (EnergyExtension)(compResource.getExtensionList().getExtension(ExtensionType.ENERGY_EXTENSION));
504	if(resourceController.getComputingResources().contains(compResource)) {
505	if( een != null && een.getPowerProfile() != null && een.getPowerProfile().getEnergyEstimationPlugin() != null){
506	accStats.meanEnergyUsage.add(energyUsage.getSumValue());
507	}
508
509	} else if( een != null && een.getPowerProfile() != null && een.getPowerProfile().getEnergyEstimationPlugin() != null){
510	ComputingResource parent = compResource.getParent();
511	boolean top = true;
512	while(parent != null){
513	een = (EnergyExtension)(parent.getExtensionList().getExtension(ExtensionType.ENERGY_EXTENSION));
514	if(een != null && een.getPowerProfile() != null && een.getPowerProfile().getEnergyEstimationPlugin() != null) {
515	top = false;
516	break;
517	}
518	parent = parent.getParent();
519	}
520	if(top == true){
521	accStats.meanEnergyUsage.add(energyUsage.getSumValue());
522	}
523	}
524	if (energyStatsFile != null) {
525	Object txt = energyUsage.serialize(serializer);
526	energyStatsFile.print(txt);
527	}
528
529	for(Device device: compResource.getResourceCharacteristic().getDevices()){
530	DCwormsAccumulator devAccumulator = new DCwormsAccumulator();
531	ResourcePowerStats deviceEnergyUsage = gatherResourcePowerConsumptionStats(device);
532	deviceEnergyUsage.setMeanValue(calculateMeanValue(deviceEnergyUsage));
533	deviceEnergyUsage.setSumValue(deviceEnergyUsage.getMeanValue() * (endSimulationTime - startSimulationTime) / (3600 * MILLI_SEC));
534	devAccumulator.add(deviceEnergyUsage.getSumValue());
535	//TODO - type and name may be the same (results are 2 times bigger)
536	metCalc.addMetricsData(device.getType().getName(), devAccumulator);
537	metCalc.addMetricsData(device.getFullName(), devAccumulator);
538	if (deviceStatsFile != null) {
539	Object txt = deviceEnergyUsage.serialize(serializer);
540	deviceStatsFile.print(txt);
541	}
542
543	if(ArrayUtils.contains(configuration.compResForEnergyChart, device.getType().getName())){
544	if (configuration.creatediagrams_respowerusage) {
545	createResourceEnergyDiagramData(deviceEnergyUsage);
546	}
547	}
548	}
549	}
550
551	if(type_stats.get(resourceTypeName).contains(Stats.chartEnergy)){
552	if (configuration.creatediagrams_respowerusage) {
553	createResourceEnergyDiagramData(energyUsage);
554	}
555	}
556
557	if(type_stats.get(resourceTypeName).contains(Stats.textAirflow)){
558	airflow = gatherResourceAirflowStats(compResource);
559	airflow.setMeanValue(calculateMeanValue(airflow));
560	airflow.setSumValue(airflow.getMeanValue() * (endSimulationTime - startSimulationTime) / (60 * MILLI_SEC));
561
562	EnergyExtension een = (EnergyExtension)(compResource.getExtensionList().getExtension(ExtensionType.ENERGY_EXTENSION));
563	if(resourceController.getComputingResources().contains(compResource)) {
564	if( een != null && een.getAirflowProfile() != null && een.getPowerProfile().getEnergyEstimationPlugin() != null){
565	accStats.meanAirflow.add(airflow.getMeanValue());
566	}
567
568	} else if( een != null && een.getAirflowProfile() != null ){
569	ComputingResource parent = compResource.getParent();
570	een = (EnergyExtension)(parent.getExtensionList().getExtension(ExtensionType.ENERGY_EXTENSION));
571	boolean top = true;
572	while(parent != null){
573	if(een != null && een.getAirflowProfile() != null) {
574	top = false;
575	break;
576	}
577	parent = parent.getParent();
578	}
579	if(top == true){
580	accStats.meanAirflow.add(airflow.getMeanValue());
581	}
582	}
583	if (airflowStatsFile != null) {
584	Object txt = airflow.serialize(serializer);
585	airflowStatsFile.print(txt);
586	}
587
588	for(Device device: compResource.getResourceCharacteristic().getDevices()){
589	ResourceAirflowStats deviceAirflow = gatherResourceAirflowStats(device);
590	deviceAirflow.setMeanValue(calculateMeanValue(deviceAirflow));
591	deviceAirflow.setSumValue(deviceAirflow.getMeanValue() * (endSimulationTime - startSimulationTime) / (60 * MILLI_SEC));
592
593	if (deviceStatsFile != null) {
594	Object txt = deviceAirflow.serialize(serializer);
595	deviceStatsFile.print(txt);
596	}
597
598	if(ArrayUtils.contains(configuration.compResForAirflowChart, device.getType().getName())){
599	if (configuration.creatediagrams_resairflow) {
600	createResourceAirflowDiagramData(deviceAirflow);
601	}
602	}
603	}
604	}
605
606	if(type_stats.get(resourceTypeName).contains(Stats.chartAirflow)){
607
608	if (configuration.creatediagrams_resairflow) {
609	createResourceAirflowDiagramData(airflow);
610	}
611	}
612
613	if(type_stats.get(resourceTypeName).contains(Stats.textTemperature)){
614	temperature = gatherResourceTemperatureStats(compResource);
615	temperature.setMeanValue(calculateMeanValue(temperature));
616	temperature.setSumValue(temperature.getMeanValue() * (endSimulationTime - startSimulationTime) / (3600 * MILLI_SEC));
617	/*if(resourceTypeName.equals("Processor")){
618	tempAboveThresholdAccumulator.add(calculateTempExcTime(temperature));
619	tempAboveThresholdAccumulatorH.add(calculateTempExcTimeH(temperature));
620	}*/
621	EnergyExtension een = (EnergyExtension)(compResource.getExtensionList().getExtension(ExtensionType.ENERGY_EXTENSION));
622	if(resourceController.getComputingResources().contains(compResource)) {
623	if( een != null && een.getThermalProfile() != null && een.getPowerProfile().getEnergyEstimationPlugin() != null){
624	accStats.meanTemperature.add(temperature.getMeanValue());
625	}
626
627	} else if( een != null && een.getThermalProfile() != null ){
628	ComputingResource parent = compResource.getParent();
629	een = (EnergyExtension)(parent.getExtensionList().getExtension(ExtensionType.ENERGY_EXTENSION));
630	boolean top = true;
631	while(parent != null){
632	if(een != null && een.getThermalProfile() != null) {
633	top = false;
634	break;
635	}
636	parent = parent.getParent();
637	}
638	if(top == true){
639	accStats.meanTemperature.add(temperature.getMeanValue());
640	}
641	}
642	if (temperatureStatsFile != null) {
643	Object txt = temperature.serialize(serializer);
644	temperatureStatsFile.print(txt);
645	}
646
647	for(Device device: compResource.getResourceCharacteristic().getDevices()){
648	ResourceTemperatureStats deviceTemperature = gatherResourceTemperatureStats(device);
649	deviceTemperature.setMeanValue(calculateMeanValue(deviceTemperature));
650	deviceTemperature.setSumValue(deviceTemperature.getMeanValue() * (endSimulationTime - startSimulationTime) / (3600 * MILLI_SEC));
651
652	if (deviceStatsFile != null) {
653	Object txt = deviceTemperature.serialize(serializer);
654	deviceStatsFile.print(txt);
655	}
656
657	if(ArrayUtils.contains(configuration.compResForTemperatureChart, device.getType().getName())){
658	if (configuration.creatediagrams_restemperature) {
659	createResourceTemperatureDiagramData(deviceTemperature);
660	}
661	}
662	}
663	}
664
665	if(type_stats.get(resourceTypeName).contains(Stats.chartTemperature)){
666	if (configuration.creatediagrams_restemperature) {
667	createResourceTemperatureDiagramData(temperature);
668	}
669	}
670
671	if(type_stats.get(resourceTypeName).contains(Stats.textLoad)){
672	usefulWork = gatherResourceUsefulWorkStats(compResource);
673	if (usefulWorkStatsFile != null) {
674	Object txt = usefulWork.serialize(serializer);
675	usefulWorkStatsFile.print(txt);
676	}
677	}
678	/*if (nodesAvailabilityStatsFile != null) {
679	ResourceAvailabilityStats ras = new ResourceAvailabilityStats(compResource.getFullName(), compResource.getType(), "availableNodes", compResource.getChildren().size(), endSimulationTime);
680	Object txt = ras.serialize(serializer);
681	nodesAvailabilityStatsFile.print(txt);
682	ras = gatherResourceAvailabilityStats(compResource);
683	txt = ras.serialize(serializer);
684	nodesAvailabilityStatsFile.print(txt);
685	}*/
686
687	}
688	metCalc.addMetricsData(resourceTypeName + "_CALC", calculationsEnergyAccumulator);
689	metCalc.addMetricsData(resourceTypeName, resourceEnergyAccumulator);
690	metCalc.addMetricsData(resourceTypeName + "_MAX", maxResourceEnergyAccumulator);
691	/*if(resourceTypeName.equals("Processor")){
692	metCalc.addMetricsData(resourceTypeName + "_TEMPEXC", tempAboveThresholdAccumulator);
693	metCalc.addMetricsData(resourceTypeName + "_TEMPHEXC", tempAboveThresholdAccumulatorH);
694	}*/
695	}
696	}
697
698
699	if (metricsStatsFile != null) {
700	//log.info("#STATS " + "Metrics");
701	System.out.println("#STATS " + "===== Metrics =====");
702	for(MetricsStats metric: metCalc.calulateMetrics()){
703	Object txt = metric.serialize(serializer);
704	//log.info("#STATS " + metric.getResourceName() + " " + metric.getMetricName() + ": " + metric.getValue());
705	DecimalFormat df = new DecimalFormat();
706	df.setMaximumFractionDigits(3);
707	df.setGroupingUsed(false);
708	System.out.println("#STATS " + metric.getMetricName().replaceAll("_", " ") + ": " + df.format(metric.getValue()) + " [" + metric.getUnit() + "]");
709	metricsStatsFile.println(txt);
710	}
711	}
712	saveResourceDiagrams();
713	createAccumulatedResourceSimulationStatistic();
714
715	if (temperatureStatsFile != null) {
716	temperatureStatsFile.close();
717	}
718	if (airflowStatsFile != null) {
719	airflowStatsFile.close();
720	}
721	if (energyStatsFile != null) {
722	energyStatsFile.close();
723	}
724	if (resourceOccupancyStatsFile != null) {
725	resourceOccupancyStatsFile.close();
726	}
727	if (resourceUtilizationStatsFile != null) {
728	resourceUtilizationStatsFile.close();
729	}
730
731	if (usefulWorkStatsFile != null) {
732	usefulWorkStatsFile.close();
733	}
734	if (deviceStatsFile != null) {
735	deviceStatsFile.close();
736	}
737
738	/*if (nodesAvailabilityStatsFile != null) {
739	nodesAvailabilityStatsFile.close();
740	}*/
741
742	if (metricsStatsFile != null) {
743	metricsStatsFile.close();
744	}
745
746	}
747
748
749	/*private double calculateTempExcTime(ResourceTemperatureStats temperature) {
750	double overall = 0;
751	long prevt = 0;
752	//double prevTemp;
753	boolean exceeded = false;
754	for(Long t: temperature.getHistory().keySet()){
755	if(exceeded){
756	overall = overall + (t - prevt);
757	exceeded = false;
758	}
759	double temp = temperature.getHistory().get(t);
760	if(temp > EnvironmentConditions.i5_statsTThres){
761	prevt = t;
762	exceeded = true;
763	//prevTemp = temp;
764	}
765	}
766	return overall;
767	}
768	private double calculateTempExcTimeH(ResourceTemperatureStats temperature) {
769	double overall = 0;
770	long prevt = 0;
771	//double prevTemp;
772	boolean exceeded = false;
773	for(Long t: temperature.getHistory().keySet()){
774	if(exceeded){
775	overall = overall + (t - prevt);
776	exceeded = false;
777	}
778	double temp = temperature.getHistory().get(t);
779	if(temp > EnvironmentConditions.i5_statsTThresH){
780	prevt = t;
781	exceeded = true;
782	//prevTemp = temp;
783	}
784	}
785	return overall;
786	}*/
787
788	private void peStatsPostProcessing(Map<ComputingResource, List<ResStat>> basicResStats){
789	ResourceType resType = null;
790
791	for(ComputingResource pe: basicResStats.keySet()){
792	List<ResStat> resStats = basicResStats.get(pe);
793	resType = resStats.get(0).getResType();
794	break;
795	}
796	List<ComputingResource> resources = null;
797	try {
798	resources = new ArrayList<ComputingResource>();
799	for(ComputingResource compRes: resourceController.getComputingResources()){
800	resources.addAll(compRes.getDescendantsByType(resType));
801	}
802	} catch (Exception e) {
803	return;
804	}
805	for(ComputingResource resource: resources){
806	if(!basicResStats.containsKey(resource)){
807	basicResStats.put(resource, new ArrayList<ResStat>());
808	}
809	}
810	}
811
812	//TO DELETE
813	/*private Map<ComputingResource, List<ResStat>> gatherPEStats(ExecutablesList executables) {
814
815	Map<ComputingResource, List<ResStat>> basicResStats = new TreeMap<ComputingResource, List<ResStat>>(new peIdComparator());
816	for (ExecTask execTask:executables) {
817	Executable exec = (Executable) execTask;
818
819
820	LinkedList<ResourceItem> resourceHistory = exec.getAllocatedResources();
821	if(resourceHistory.size() == 0)
822	continue;
823	Map<ResourceUnitName, ResourceUnit> res = exec.getAllocatedResources().getLast().getResourceUnits();
824	ResourceUnit resUnit = res.get(StandardResourceUnitName.PE);
825	//ProcessingElements pes = (ProcessingElements) resUnit ;
826	if(resUnit instanceof ProcessingElements){
827	ProcessingElements pes = (ProcessingElements) resUnit;
828	for(ComputingResource pe: pes){
829	String peName = pe.getFullName();
830	long startDate = Double.valueOf(exec.getExecStartTime()).longValue() * MILLI_SEC;
831	long endDate = Double.valueOf(exec.getFinishTime()).longValue() * MILLI_SEC;
832
833	String appID = getAppId(execTask);
834
835	ResStat resStat = new ResStat(peName, pe.getType(), startDate, endDate, appID);
836
837	List<ResStat> resStats = basicResStats.get(pe);
838	if (resStats == null) {
839	resStats = new ArrayList<ResStat>();
840	resStats.add(resStat);
841	basicResStats.put(pe, resStats);
842	} else {
843	resStats.add(resStat);
844	}
845
846	String uniqueTaskID = getUniqueTaskId(execTask);
847
848	Set<ComputingResource> peNames = taskToPEMap.get(uniqueTaskID);
849	if (peNames == null) {
850	peNames = new HashSet<ComputingResource>();
851	peNames.add(pe);
852	taskToPEMap.put(uniqueTaskID, peNames);
853	} else {
854	peNames.add(pe);
855	}
856
857	try{
858	double usefulWork = execTask.getExecutionProfile().getUsefulWork()/pes.size();
859	//double usefulWork = gatherUsefulWorkStats(pe).getValue();
860	DCwormsAccumulator uwAcc;
861	if(metCalc.getMetricsData().containsKey("UW_" + pe.getFullName())){
862	uwAcc = metCalc.getMetricsData().get("UW_" + pe.getFullName()).get(0);
863	uwAcc.add(usefulWork);
864	} else {
865	uwAcc = new DCwormsAccumulator();
866	uwAcc.add(usefulWork);
867	metCalc.addMetricsData("UW_" + pe.getFullName(), uwAcc);
868	}
869	} catch (Exception e){
870
871	}
872
873	}
874	}
875	}
876	return basicResStats;
877	}*/
878
879	private Map<ComputingResource, List<ResStat>> gatherPEStatsMulti(ExecutablesList executables) {
880
881	Map<ComputingResource, List<ResStat>> basicResStats = new TreeMap<ComputingResource, List<ResStat>>(new peIdComparator());
882	for (ExecTask execTask:executables) {
883	Executable exec = (Executable) execTask;
884
885	LinkedList<ResourceItem> resourceHistory = exec.getAllocatedResources();
886	if(resourceHistory.size() == 0)
887	continue;
888
889	for(int i = 0; i < resourceHistory.size(); i++){
890	ResourceItem resHistItem = resourceHistory.get(i);
891	Map<ResourceUnitName, ResourceUnit> res = resHistItem.getResourceUnits();
892	ResourceUnit resUnit = res.get(StandardResourceUnitName.PE);
893	//ProcessingElements pes = (ProcessingElements) resUnit ;
894	LinkedList<ExecutionHistoryItem> execHistory = exec.getExecutionHistory();
895	long st = -1, et;
896	for(int j = 0; j < execHistory .size(); j++){
897	ExecutionHistoryItem execHistItem = execHistory.get(j);
898	if(execHistItem.getStatus() == DCWormsTags.PAUSED \|\| execHistItem.getResIndex() != i){
899	continue;
900	}
901	if(st == -1){
902	st = execHistItem.getTimeStamp();
903	}
904
905	if(j < execHistory.size() - 1 && (execHistory.get(j + 1).getResIndex() != execHistItem.getResIndex() \|\| execHistory.get(j + 1).getStatus() == DCWormsTags.PAUSED) \|\| j + 1 == execHistory.size()){
906	if(j < execHistory.size() -1 && (execHistory.get(j + 1).getResIndex() != execHistItem.getResIndex() \|\| execHistory.get(j + 1).getStatus() == DCWormsTags.PAUSED))
907	et = execHistory.get(j + 1).getTimeStamp();
908	else et = Double.valueOf(exec.getFinishTime()).longValue() * MILLI_SEC;
909	if(resUnit instanceof ProcessingElements){
910	ProcessingElements pes = (ProcessingElements) resUnit;
911	for(ComputingResource pe: pes){
912	String peName = pe.getFullName();
913	long startDate = st;
914	long endDate;
915	if(i == resourceHistory.size() - 1){
916	endDate = Double.valueOf(exec.getFinishTime()).longValue() * MILLI_SEC;
917	}else {
918	endDate = et;
919	}
920
921	String appID = getAppId(execTask);
922
923	ResStat resStat = new ResStat(peName, pe.getType(), startDate, endDate, appID);
924
925	List<ResStat> resStats = basicResStats.get(pe);
926	if (resStats == null) {
927	resStats = new ArrayList<ResStat>();
928	resStats.add(resStat);
929	basicResStats.put(pe, resStats);
930	} else {
931	resStats.add(resStat);
932	}
933
934	String uniqueTaskID = getUniqueTaskId(execTask);
935
936	Set<ComputingResource> peNames = taskToPEMap.get(uniqueTaskID);
937	if (peNames == null) {
938	peNames = new HashSet<ComputingResource>();
939	peNames.add(pe);
940	taskToPEMap.put(uniqueTaskID, peNames);
941	} else {
942	peNames.add(pe);
943	}
944
945	try{
946	double usefulWork = execTask.getExecutionProfile().getUsefulWork()/pes.size();
947	usefulWork = ((et - st) / (1000 * getExecutionTime(execTask))) * usefulWork;
948	DCwormsAccumulator uwAcc;
949	if(metCalc.getMetricsData().containsKey("UW_" + pe.getFullName())){
950	uwAcc = metCalc.getMetricsData().get("UW_" + pe.getFullName()).get(0);
951	uwAcc.add(usefulWork);
952	} else {
953	uwAcc = new DCwormsAccumulator();
954	uwAcc.add(usefulWork);
955	metCalc.addMetricsData("UW_" + pe.getFullName(), uwAcc);
956	}
957	} catch (Exception e){
958
959	}
960	}
961	}
962	st = -1;
963	}
964	}
965	}
966	}
967	return basicResStats;
968	}
969
970	private double getExecutionTime(ExecTask execTask) {
971	double executionTime = 0;
972	long previousTimestamp = 0;
973	int previousStatus = DCWormsTags.CREATED;
974	for(ExecutionHistoryItem execHistItem: execTask.getExecutionHistory()){
975	if(previousStatus == DCWormsTags.INEXEC){
976	executionTime = executionTime + (execHistItem.getTimeStamp()/1000 - previousTimestamp);
977	}
978	previousTimestamp = execHistItem.getTimeStamp()/1000;
979	previousStatus = execHistItem.getStatus();
980	}
981	return executionTime;
982	//return task.getFinishTime() - task.getExecStartTime();
983	}
984
985	private String getUniqueTaskId(ExecTask execTask){
986	return execTask.getJobId() + "_" + execTask.getId();
987	}
988
989	//change this method to adjust the colors of gantt chart
990	protected String getAppId(ExecTask execTask){
991	String taskId = execTask.getJobId() + "_" + execTask.getId();
992	if(execTask.getApplicationName() != null){
993	taskId = execTask.getApplicationName();
994	}
995	return taskId;
996	}
997
998	//TO DELETE
999	/*private ResourceUsageStats gatherResourceOccupancyStatsOld(ComputingResource resource, Map<String, List<ResStat>> basicStats) {
1000	String usageType = null;
1001	if(resource.getResourceCharacteristic().getParameters().get("load-sensor") != null){
1002	usageType = resource.getResourceCharacteristic().getParameters().get("load-sensor").get(0).getContent();
1003	}
1004	ResourceUsageStats usageStats = new ResourceUsageStats(resource.getFullName(), resource.getType(), usageType);
1005	int cnt = 0;
1006	System.out.println("a: " + System.currentTimeMillis());
1007	for(String resName: basicStats.keySet()){
1008
1009	if(resource.getDescendantByName(resName) != null \|\| resource.getFullName().compareTo(resName) == 0){
1010	createResourceLoadData(usageStats, basicStats.get(resName));
1011	cnt++;
1012	}
1013	}
1014	System.out.println("b: " + System.currentTimeMillis());
1015	for(Long key: usageStats.getHistory().keySet()){
1016	Double value = usageStats.getHistory().get(key)/cnt;
1017	usageStats.getHistory().put(key, value);
1018	}
1019
1020	return usageStats;
1021	}
1022	*/
1023	private ResourceUsageStats gatherResourceOccupancyStats(ComputingResource compRes, Map<ComputingResource, List<ResStat>> basicStats) {
1024
1025	String usageType = null;
1026	if(compRes.getResourceCharacteristic().getParameters().get("load-sensor")!= null){
1027	usageType = compRes.getResourceCharacteristic().getParameters().get("load-sensor").get(0).getContent();
1028	}
1029	ResourceUsageStats usageStats = new ResourceUsageStats(compRes.getFullName(), compRes.getType(), usageType);
1030	int cnt = 0;
1031
1032	if(basicStats.containsKey(compRes)){
1033	createResourceLoadData(usageStats, basicStats.get(compRes));
1034	cnt++;
1035	} else {
1036	for(ComputingResource pe: basicStats.keySet()){
1037	if(compRes.contains(pe)){
1038	createResourceLoadData(usageStats, basicStats.get(pe));
1039	cnt++;
1040	}
1041	}
1042	}
1043
1044	for(Long key: usageStats.getHistory().keySet()){
1045	Double value = usageStats.getHistory().get(key)/cnt;
1046	usageStats.getHistory().put(key, value);
1047	}
1048	return usageStats;
1049	}
1050
1051	private ResourceAvailabilityStats gatherResourceAvailabilityStats(ComputingResource resource) {
1052	String usageType = new String ("usedNodes");
1053
1054	int usedResources = 0;
1055	for(ComputingResource compResource: resource.getChildren()){
1056	double meanLoad = calculateMeanValue(gatherResourceOccupancyStats(compResource, peStats));
1057	if(meanLoad > 0){
1058	usedResources++;
1059	}
1060	}
1061	ResourceAvailabilityStats availabilityStats = new ResourceAvailabilityStats(resource.getFullName(), resource.getType(), usageType, usedResources, endSimulationTime);
1062	return availabilityStats;
1063	}
1064
1065	private ResourceUsefulWorkStats gatherResourceUsefulWorkStats(ComputingResource compResource) {
1066	String usageType = "usefulWork";
1067	double usefulWork = 0;
1068	JobRegistry jr = new JobRegistryImpl(compResource);
1069	for(ExecTask task: jr.getFinishedTasks()){
1070	usefulWork = usefulWork + task.getExecutionProfile().getUsefulWork();
1071	}
1072	ResourceUsefulWorkStats usefulWorkStats = new ResourceUsefulWorkStats(compResource.getFullName(), compResource.getType(), usageType, usefulWork, endSimulationTime);
1073	return usefulWorkStats;
1074	}
1075
1076	private Map<Long, Double> createResourceLoadData(ResourceUsageStats usageStats, List<ResStat> resStats) {
1077
1078	TreeMap<Long, Double> ganttData = (TreeMap<Long, Double>)usageStats.getHistory();
1079	for (ResStat resStat : resStats) {
1080
1081	long start_time = resStat.getStartDate();
1082	long end_time = resStat.getEndDate();
1083	if (end_time > start_time) {
1084	Double end = ganttData.get(end_time);
1085	if (end == null) {
1086	ganttData.put(end_time, 0.0);
1087	Long left = getLeftKey(ganttData, end_time);
1088	if (left != null) {
1089	Double leftValue = ganttData.get(left);
1090	ganttData.put(end_time, leftValue);
1091	}
1092	}
1093
1094	Double start = ganttData.get(start_time);
1095	if (start == null) {
1096	ganttData.put(start_time, 0.0);
1097	Long left = getLeftKey(ganttData, start_time);
1098	if (left != null) {
1099	Double leftValue = ganttData.get(left);
1100	ganttData.put(start_time, leftValue);
1101	}
1102	}
1103
1104	SortedMap<Long, Double> sm = ganttData.subMap(start_time,
1105	end_time);
1106	for (Long key : sm.keySet()) {
1107	Double keyVal = ganttData.get(key);
1108	ganttData.put(key, keyVal + 1);
1109	}
1110	}
1111	}
1112	if(ganttData.get(startSimulationTime) == null)
1113	ganttData.put(startSimulationTime, 0.0);
1114	if(ganttData.get(endSimulationTime) == null)
1115	ganttData.put(endSimulationTime, 0.0);
1116	return ganttData;
1117	}
1118
1119	private Long getLeftKey(TreeMap<Long, Double> ganttData, Long toKey) {
1120
1121	SortedMap<Long, Double> sm = ganttData.headMap(toKey);
1122	if (sm.isEmpty()) {
1123	return null;
1124	}
1125	return sm.lastKey();
1126	}
1127
1128	private ResourceUsageStats gatherResourceUtlizationStats(PhysicalResource resource) {
1129
1130	String usageType = null;
1131	if(resource.getResourceCharacteristic().getParameters().get("load-sensor")!= null){
1132	usageType = resource.getResourceCharacteristic().getParameters().get("load-sensor").get(0).getContent();
1133	}
1134	ResourceUsageStats resUtlization = new ResourceUsageStats(resource.getFullName(), resource.getType(), usageType);
1135	Map<Long, Double> usage = resUtlization.getHistory();
1136
1137	ExtensionList extensionList = resource.getExtensionList();
1138	if(extensionList != null){
1139	for (Extension extension : extensionList) {
1140	if (extension.getType() == ExtensionType.LOAD_EXTENSION) {
1141	LoadExtension le = (LoadExtension)extension;
1142	if(le.getLoadProfile() == null)
1143	break;
1144	List<MeasurementHistory> loadHistory = le.getLoadProfile().getUtilizationHistory();
1145	if(loadHistory.size() == 0)
1146	break;
1147	long endSimulationTime = DateTimeUtilsExt.currentTimeMillis();
1148
1149	loadHistory.add(new MeasurementHistory(endSimulationTime, le.getLoadProfile().getUtilizationHistory().get(le.getLoadProfile().getUtilizationHistory().size() - 1).getValue()));
1150	for(MeasurementHistory l:loadHistory){
1151	usage.put(l.getTimestamp(), l.getValue()/100);
1152	}
1153	}
1154	}
1155	}
1156	return resUtlization;
1157	}
1158
1159	private ResourcePowerStats gatherResourcePowerConsumptionStats(PhysicalResource resource) {
1160	double power = 0;
1161
1162	String usageType = null;
1163	if(resource.getResourceCharacteristic().getParameters().get("power-sensor") != null){
1164	usageType = resource.getResourceCharacteristic().getParameters().get("power-sensor").get(0).getContent();
1165	}
1166	ResourcePowerStats resEnergyUsage = new ResourcePowerStats(resource.getFullName(), resource.getType(), usageType);
1167	Map<Long, Double> usage = resEnergyUsage.getHistory();
1168
1169	ExtensionList extensionList = resource.getExtensionList();
1170	if(extensionList != null){
1171	for (Extension extension : extensionList) {
1172	if (extension.getType() == ExtensionType.ENERGY_EXTENSION) {
1173	EnergyExtension ee = (EnergyExtension)extension;
1174	if(ee.getPowerProfile() == null)
1175	break;
1176	List<PowerUsage> powerUsage = ee.getPowerProfile().getPowerUsageHistory();
1177	if(powerUsage.size() == 0)
1178	break;
1179	long lastTime = DateTimeUtilsExt.getOffsetTime().getTimeInMillis();
1180	long endSimulationTime = DateTimeUtilsExt.currentTimeMillis();
1181	double lastPower = 0;
1182	powerUsage.add(new PowerUsage(endSimulationTime, ee.getPowerProfile().getPowerUsageHistory().get(ee.getPowerProfile().getPowerUsageHistory().size()-1).getValue()));
1183	for(PowerUsage pu:powerUsage){
1184	usage.put(pu.getTimestamp(), pu.getValue());
1185
1186	power = power + (pu.getTimestamp() - lastTime) * lastPower/ (3600 * MILLI_SEC);
1187	lastPower = pu.getValue();
1188	lastTime = pu.getTimestamp();
1189	}
1190	}
1191	}
1192	}
1193	return resEnergyUsage;
1194	}
1195
1196	private ResourceAirflowStats gatherResourceAirflowStats(PhysicalResource resource) {
1197
1198	String usageType = null;
1199	if(resource.getResourceCharacteristic().getParameters().get("airflow_volume-sensor") != null){
1200	usageType = resource.getResourceCharacteristic().getParameters().get("airflow_volume-sensor").get(0).getContent();
1201	}
1202	ResourceAirflowStats resAirflow = new ResourceAirflowStats(resource.getFullName(), resource.getType(), usageType);
1203	Map<Long, Double> airflow = resAirflow.getHistory();
1204
1205	ExtensionList extensionList = resource.getExtensionList();
1206	if(extensionList != null){
1207	for (Extension extension : extensionList) {
1208	if (extension.getType() == ExtensionType.ENERGY_EXTENSION) {
1209	EnergyExtension ee = (EnergyExtension)extension;
1210	if(ee.getAirflowProfile() == null)
1211	break;
1212	List<AirflowValue> airflowHistory = ee.getAirflowProfile().getAirflowHistory();
1213	if(airflowHistory.size() == 0)
1214	break;
1215	long endSimulationTime = DateTimeUtilsExt.currentTimeMillis();
1216	airflowHistory.add(new AirflowValue(endSimulationTime, ee.getAirflowProfile().getAirflowHistory().get(ee.getAirflowProfile().getAirflowHistory().size()-1).getValue()));
1217	for(AirflowValue af:airflowHistory){
1218	airflow.put(af.getTimestamp(), af.getValue());
1219	}
1220	}
1221	}
1222	}
1223	return resAirflow;
1224	}
1225
1226	private ResourceTemperatureStats gatherResourceTemperatureStats(PhysicalResource resource) {
1227
1228	String usageType = null;
1229	if(resource.getResourceCharacteristic().getParameters().get("temperature-sensor")!= null){
1230	usageType = resource.getResourceCharacteristic().getParameters().get("temperature-sensor").get(0).getContent();
1231	}
1232	ResourceTemperatureStats resTemperature = new ResourceTemperatureStats(resource.getFullName(), resource.getType(), usageType);
1233	Map<Long, Double> temperature = resTemperature.getHistory();
1234
1235	ExtensionList extensionList = resource.getExtensionList();
1236	if(extensionList != null){
1237	for (Extension extension : extensionList) {
1238	if (extension.getType() == ExtensionType.ENERGY_EXTENSION) {
1239	EnergyExtension ee = (EnergyExtension)extension;
1240	if(ee.getThermalProfile() == null)
1241	break;
1242
1243	List<TemperatureValue> temperatureHistory = ee.getThermalProfile().getTemperatureHistory();
1244	if(temperatureHistory.size() == 0)
1245	break;
1246	long endSimulationTime = DateTimeUtilsExt.currentTimeMillis();
1247	temperatureHistory.add(new TemperatureValue(endSimulationTime, ee.getThermalProfile().getTemperatureHistory().get(ee.getThermalProfile().getTemperatureHistory().size()-1).getValue()));
1248	for(TemperatureValue af: temperatureHistory){
1249	temperature.put(af.getTimestamp(), af.getValue());
1250	}
1251	}
1252	}
1253	}
1254	return resTemperature;
1255	}
1256
1257	private void createResourceOccupancyDiagram(ResourceUsageStats resLoadStats) {
1258
1259	for(Long key: resLoadStats.getHistory().keySet()){
1260	Double value = resLoadStats.getHistory().get(key) * 100;
1261	resLoadStats.getHistory().put(key, value);
1262	}
1263
1264	XYDataset dataset = createResourceChartDataSet(resLoadStats,
1265	startSimulationTime, endSimulationTime);
1266
1267	List<XYDataset> occupancyDiagram = resourceOccupancyDiagrams.get(resLoadStats.getResourceType().getName());
1268	if(occupancyDiagram == null){
1269	occupancyDiagram = new ArrayList<XYDataset>();
1270	occupancyDiagram.add(dataset);
1271	resourceOccupancyDiagrams.put(resLoadStats.getResourceType().getName(), occupancyDiagram);
1272	} else {
1273	occupancyDiagram.add(dataset);
1274	}
1275	}
1276
1277	private void createResourceUtilizationDiagram(ResourceUsageStats resLoadStats) {
1278
1279	for(Long key: resLoadStats.getHistory().keySet()){
1280	Double value = resLoadStats.getHistory().get(key) * 100;
1281	resLoadStats.getHistory().put(key, value);
1282	}
1283
1284	XYDataset dataset = createResourceChartDataSet(resLoadStats,
1285	startSimulationTime, endSimulationTime);
1286
1287	List<XYDataset> utilizationDiagram = resourceUtilizationDiagrams.get(resLoadStats.getResourceType().getName());
1288	if(utilizationDiagram == null){
1289	utilizationDiagram = new ArrayList<XYDataset>();
1290	utilizationDiagram.add(dataset);
1291	resourceUtilizationDiagrams.put(resLoadStats.getResourceType().getName(), utilizationDiagram);
1292	} else {
1293	utilizationDiagram.add(dataset);
1294	}
1295	}
1296
1297	private void createResourceEnergyDiagramData(ResourceDynamicStats powerConsumptionStats) {
1298
1299	XYDataset dataset = createResourceChartDataSet(powerConsumptionStats,
1300	startSimulationTime, endSimulationTime);
1301
1302	List<XYDataset> energyDiagramData = resourcePowerUsageDiagrams.get(powerConsumptionStats.getResourceType().getName());
1303	if(energyDiagramData == null){
1304	energyDiagramData = new ArrayList<XYDataset>();
1305	energyDiagramData.add(dataset);
1306	resourcePowerUsageDiagrams.put(powerConsumptionStats.getResourceType().getName(), energyDiagramData);
1307	} else {
1308	energyDiagramData.add(dataset);
1309	}
1310	}
1311
1312	private void createResourceTemperatureDiagramData(ResourceDynamicStats temperatureStats) {
1313
1314	XYDataset dataset = createResourceChartDataSet(temperatureStats,
1315	startSimulationTime, endSimulationTime);
1316
1317	List<XYDataset> temperatureDiagramData = resourceTemperatureDiagrams.get(temperatureStats.getResourceType().getName());
1318	if(temperatureDiagramData == null){
1319	temperatureDiagramData = new ArrayList<XYDataset>();
1320	temperatureDiagramData.add(dataset);
1321	resourceTemperatureDiagrams.put(temperatureStats.getResourceType().getName(), temperatureDiagramData);
1322	} else {
1323	temperatureDiagramData.add(dataset);
1324	}
1325	}
1326
1327	private void createResourceAirflowDiagramData(ResourceDynamicStats airflowStats) {
1328
1329	XYDataset dataset = createResourceChartDataSet(airflowStats,
1330	startSimulationTime, endSimulationTime);
1331
1332	List<XYDataset> airflowDiagramData = resourceAirflowDiagrams.get(airflowStats.getResourceType().getName());
1333	if(airflowDiagramData == null){
1334	airflowDiagramData = new ArrayList<XYDataset>();
1335	airflowDiagramData.add(dataset);
1336	resourceAirflowDiagrams.put(airflowStats.getResourceType().getName(), airflowDiagramData);
1337	} else {
1338	airflowDiagramData.add(dataset);
1339	}
1340	}
1341
1342	private XYDataset createResourceChartDataSet(
1343	ResourceDynamicStats dynamicStats, long start, long end) {
1344
1345	Map<Long, Double> chartData = getResourceChartRawData(dynamicStats);
1346	XYSeriesCollection dataset = new XYSeriesCollection();
1347	XYSeries data = new XYSeries(dynamicStats.getResourceName(), false, true);
1348	data.add(start, 0);
1349	for (Long key : chartData.keySet()) {
1350	Double val = chartData.get(key);
1351	if(key >= startSimulationTime)
1352	data.add(key, val);
1353	}
1354	data.add(end, 0);
1355	dataset.addSeries(data);
1356	return dataset;
1357	}
1358
1359	private Map<Long, Double> getResourceChartRawData(
1360	ResourceDynamicStats dynamicStats) {
1361
1362	Map<Long, Double> history = dynamicStats.getHistory();
1363	Map<Long, Double> chartData = new TreeMap<Long, Double>();
1364	for (Long key: history.keySet()) {
1365	chartData.put(key, history.get(key));
1366	}
1367	return chartData;
1368	}
1369
1370	private boolean saveResourceDiagrams() {
1371
1372	if (!generateDiagrams)
1373	return false;
1374
1375	String fileName = new File(outputFolderName, DIAGRAMS_FILE_NAME_PREFIX
1376	+ simulationIdentifier + "_").getAbsolutePath();
1377
1378	String chartName = "Gantt diagram for "
1379	+ DataCenterWorkloadSimulator.SIMULATOR_NAME;
1380	String simulationTime = "Simulation time";
1381	Title subtitle = new TextTitle("created for \"" + simulationIdentifier
1382	+ "\" at " + Calendar.getInstance().getTime().toString());
1383
1384	JFreeChart peDiagram = null;
1385	if (configuration.creatediagrams_gantt) {
1386	peDiagram = getPEGanttDiagram(chartName, subtitle,
1387	simulationTime);
1388	if (!saveCategoryChart(peDiagram, fileName + "Gantt",
1389	"{0}"))
1390	return false;
1391	}
1392	chartName = "Load diagram for "
1393	+ DataCenterWorkloadSimulator.SIMULATOR_NAME;
1394	JFreeChart resourceOccupancyDiagram = null;
1395	if (configuration.creatediagrams_resutilization) {
1396	String axisName = "OCCUPANCY [%]";
1397	for(String resType: resourceOccupancyDiagrams.keySet()){
1398	resourceOccupancyDiagram = getResourceDynamicDiagram(resourceOccupancyDiagrams.get(resType), simulationTime, chartName,
1399	subtitle, axisName);
1400	if (!saveXYPlotChart(resourceOccupancyDiagram, fileName + "ResourcesOccupancy_" + resType))
1401	return false;
1402	}
1403	}
1404
1405	chartName = "Utilization diagram for "
1406	+ DataCenterWorkloadSimulator.SIMULATOR_NAME;
1407	JFreeChart resourceUtilizationDiagram = null;
1408	if (configuration.creatediagrams_resutilization) {
1409	String axisName = "UTILIZATION [%]";
1410	for(String resType: resourceUtilizationDiagrams.keySet()){
1411	resourceUtilizationDiagram = getResourceDynamicDiagram(resourceUtilizationDiagrams.get(resType), simulationTime, chartName,
1412	subtitle, axisName);
1413	if (!saveXYPlotChart(resourceUtilizationDiagram, fileName + "ResourcesUtilization_" + resType))
1414	return false;
1415	}
1416	}
1417
1418	chartName = "Energy usage diagram for "
1419	+ DataCenterWorkloadSimulator.SIMULATOR_NAME;
1420	JFreeChart resourceEnergyDiagram = null;
1421	if (configuration.creatediagrams_respowerusage) {
1422	String axisName = "POWER [W]";
1423	for(String resType: resourcePowerUsageDiagrams.keySet()){
1424	resourceEnergyDiagram = getResourceDynamicDiagram(resourcePowerUsageDiagrams.get(resType), simulationTime, chartName,
1425	subtitle, axisName);
1426	if (!saveXYPlotChart(resourceEnergyDiagram, fileName + "Energy_" + resType))
1427	return false;
1428	}
1429	}
1430
1431	chartName = "Air flow diagram for "
1432	+ DataCenterWorkloadSimulator.SIMULATOR_NAME;
1433	JFreeChart resourceAirflowDiagram = null;
1434	if (configuration.creatediagrams_resairflow) {
1435	String axisName = "AIR FLOW [m^3/min]";
1436	for(String resType: resourceAirflowDiagrams.keySet()){
1437	resourceAirflowDiagram = getResourceDynamicDiagram(resourceAirflowDiagrams.get(resType), simulationTime, chartName,
1438	subtitle, axisName);
1439	if (!saveXYPlotChart(resourceAirflowDiagram, fileName + "Airflow_" + resType))
1440	return false;
1441	}
1442	}
1443
1444	chartName = "Temperature diagram for "
1445	+ DataCenterWorkloadSimulator.SIMULATOR_NAME;
1446	JFreeChart resourceTemperatureDiagram = null;
1447	if (configuration.creatediagrams_restemperature) {
1448	String axisName = "TEMPERATURE [C]";
1449	for(String resType: resourceTemperatureDiagrams.keySet()){
1450	resourceTemperatureDiagram = getResourceDynamicDiagram(resourceTemperatureDiagrams.get(resType), simulationTime, chartName,
1451	subtitle, axisName);
1452	if (!saveXYPlotChart(resourceTemperatureDiagram, fileName + "Temperature_" + resType))
1453	return false;
1454	}
1455	}
1456	return true;
1457	}
1458
1459	private JFreeChart getPEGanttDiagram(String chartName, Title subtitle,
1460	String simulationTime) {
1461	String processors = "Processing Elements";
1462	boolean tooltip = true;
1463	boolean legend = true;
1464	JFreeChart chart = TimetableChartFactory.createTimetableChart(
1465	chartName, processors, simulationTime,
1466	ganttDiagramTimetable, legend, tooltip);
1467	chart.addSubtitle(subtitle);
1468	TimetableRenderer rend = (TimetableRenderer) chart.getCategoryPlot()
1469	.getRenderer();
1470	rend.setBackgroundBarPaint(null);
1471	return chart;
1472	}
1473
1474	private JFreeChart getResourceDynamicDiagram(List<XYDataset> diagramData, String simulationTime, String chartName, Title subtitle,
1475	String axisName) {
1476	boolean urls = false;
1477	boolean tooltip = true;
1478	boolean legend = true;
1479	CombinedDomainXYPlot cPlot = new CombinedDomainXYPlot();
1480	DateAxis xAxis = new DateAxis(simulationTime);
1481	JFreeChart chart = null;
1482
1483	for (XYDataset dataset : diagramData) {
1484	JFreeChart tChart = ChartFactory.createXYStepAreaChart("", "",
1485	null, dataset, PlotOrientation.VERTICAL, legend, tooltip, urls);
1486	XYPlot tPlot = tChart.getXYPlot();
1487	NumberAxis yAxis = new NumberAxis(axisName);
1488	yAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
1489	tPlot.setRangeAxis(yAxis);
1490	XYStepAreaRenderer rend = (XYStepAreaRenderer) tPlot.getRenderer();
1491	rend.setShapesVisible(false);
1492	cPlot.add(tPlot);
1493	}
1494	cPlot.setDomainAxis(xAxis);
1495	cPlot.setDomainAxisLocation(AxisLocation.TOP_OR_LEFT);
1496	chart = new JFreeChart(chartName, cPlot);
1497	chart.addSubtitle(subtitle);
1498	return chart;
1499	}
1500
1501	private boolean saveXYPlotChart(JFreeChart c, String fileName) {
1502	c.setNotify(false);
1503	c.setAntiAlias(true);
1504	c.setTextAntiAlias(true);
1505	c.setBorderVisible(false);
1506
1507	int height = 900;
1508
1509	CombinedDomainXYPlot cPlot = (CombinedDomainXYPlot) c.getPlot();
1510
1511	/*Font fontt=new Font("Arial",Font.BOLD,26);
1512
1513	c.getTitle().setFont(fontt);
1514	Font font=new Font("Arial",0,20);
1515
1516	CombinedDomainXYPlot p = (CombinedDomainXYPlot)c.getPlot();
1517	ValueAxis daxis = p.getDomainAxis();
1518	daxis.setTickLabelFont(font);
1519	daxis.setLabelFont(font);
1520
1521	font=new Font("Arial",0,20);
1522
1523	XYPlot sp= (XYPlot) p.getSubplots().get(0);
1524	ValueAxis raxis = sp.getRangeAxis();
1525	raxis.setTickLabelFont(font);
1526	raxis.setLabelFont(font);
1527
1528	NumberAxis range = (NumberAxis) sp.getRangeAxis();
1529	range.setTickUnit(new NumberTickUnit(100));*/
1530
1531	int nPlots = cPlot.getSubplots().size();
1532
1533	if (nPlots > 3) {
1534	height = nPlots * 300;
1535	}
1536	int width = GANTT_WIDTH;
1537
1538	if (configuration.creatediagrams_resources_scale != -1)
1539	width = (int) ((endSimulationTime - startSimulationTime) * configuration.creatediagrams_resources_scale);
1540	ChartRenderingInfo info = new ChartRenderingInfo();
1541	File f = new File(fileName + "." + ImageFormat.PNG);
1542	final String dcwormsSubtitle = "Created by "
1543	+ DataCenterWorkloadSimulator.SIMULATOR_NAME
1544	+ " http://www.gssim.org/";
1545	c.addSubtitle(new TextTitle(dcwormsSubtitle));
1546	boolean encodeAlpha = false;
1547	int compression = 9;// values 0-9
1548	try {
1549	ChartUtilities.saveChartAsPNG(f, c, width, height, info,
1550	encodeAlpha, compression);
1551	} catch (IOException e) {
1552	e.printStackTrace();
1553	return false;
1554	}
1555	return true;
1556	}
1557
1558	private void createGanttDiagram(Map<ComputingResource, List<ResStat>> basicResStats) {
1559	Map<String, TimetableEventSource> peGanttMap = new HashMap<String, TimetableEventSource>();
1560	Map<String, TimetableEventGroup> taskGanttMap = new HashMap<String, TimetableEventGroup>();
1561	for(ComputingResource pe: basicResStats.keySet()){
1562	TimetableEventSource peES = new TimetableEventSource(pe.getFullName());
1563	ganttDiagramTimetable.addEventSource(peES);
1564	peGanttMap.put(pe.getFullName(), peES);
1565	for(ResStat resStat: basicResStats.get(pe)){
1566
1567	peES = peGanttMap.get(resStat.getPeName());
1568	if(peES == null){
1569	//pe = new TimetableEventSource(resStat.getPeName());
1570	// ganttDiagramPeTimetable.addEventSource(pe);
1571	// peGanttMap.put(resStat.getPeName(), pe);
1572	}
1573	TimetableEventGroup taskEG = taskGanttMap.get(resStat.getTaskID());
1574	long startDate = resStat.getStartDate();
1575	long endDate = resStat.getEndDate();
1576	if (taskEG == null) {
1577	taskEG = new TimetableEventGroup(resStat.getTaskID());
1578	taskGanttMap.put(resStat.getTaskID(), taskEG);
1579	ganttDiagramTimetable.addEventGroup(taskEG);
1580	}
1581	ganttDiagramTimetable.addEvent(peES, taskEG,
1582	new FixedMillisecond(startDate), new FixedMillisecond(endDate));
1583	}
1584	}
1585	}
1586
1587
1588	//TO DO
1589	private void createAccumulatedResourceSimulationStatistic() {
1590	Map<ComputingResource, Double> basicResLoad = calculatePELoad(peStats);
1591
1592	List<ComputingResource> resources = resourceController.getComputingResources();
1593	for(ComputingResource compRes: resources){
1594	//for(ComputingResource child :compRes.getChildren()){
1595
1596	//ResourceUsageStats resourceUsage = gatherResourceLoadStats(compRes, basicResStats);
1597	//double load = calculateMeanValue(resourceUsage);
1598	double load = calculateResourceLoad(compRes, basicResLoad);
1599	accStats.meanTotalOccupancy.add(load);
1600
1601	ResourceUsageStats resourceUtilization = gatherResourceUtlizationStats(compRes);
1602	accStats.meanTotalLoad.add(calculateMeanValue(resourceUtilization));
1603	/*ResourceEnergyStats energyUsage = gatherResourceEnergyStats(compRes);
1604	energyUsage.setMeanUsage(calculateEnergyLoad(energyUsage));
1605	accStats.meanEnergyUsage.add(energyUsage.meanUsage);*/
1606	//}
1607	}
1608	}
1609
1610	private Map<ComputingResource, Double> calculatePELoad(Map<ComputingResource, List<ResStat>> basicResStats){
1611	Map<ComputingResource, Double> peLoad = new HashMap<ComputingResource, Double>();
1612	for(ComputingResource compRes: basicResStats.keySet()){
1613	List<ResStat> resStats = basicResStats.get(compRes);
1614	double sum = 0;
1615	for(ResStat resStat:resStats){
1616	sum += (resStat.endDate - resStat.startDate);
1617	}
1618	double load = sum / (endSimulationTime - startSimulationTime);
1619	peLoad.put(compRes, load);
1620	}
1621	return peLoad;
1622	}
1623
1624	private Double calculateResourceLoad(ComputingResource compRes, Map<ComputingResource, Double> peLoad ){
1625	int peCnt = 0;
1626	double sum = 0;
1627	for(ComputingResource pe: peLoad.keySet()){
1628
1629	if(compRes.contains(pe)){
1630	Double load = peLoad.get(pe);
1631	sum += load;
1632	peCnt++;
1633	}
1634	}
1635
1636	return sum/peCnt;
1637	}
1638
1639
1640	private double calculateMeanValue(ResourceDynamicStats resDynamicStats){
1641	double meanValue = 0;
1642	long time = -1;
1643	double value = 0;
1644
1645	Map<Long, Double> history = resDynamicStats.getHistory();
1646	for (Long key : history.keySet()) {
1647
1648	if(key >= startSimulationTime){
1649	if (time != -1) {
1650	meanValue += (value * (key - time)) / (endSimulationTime - startSimulationTime);
1651	time = key;
1652	} else {
1653	time = key;
1654	}
1655	}
1656	value = history.get(key);
1657	}
1658	return meanValue;
1659	}
1660
1661
1662	private double calculateMaxValue(ResourceDynamicStats resDynamicStats){
1663	double maxValue = 0;
1664
1665	Map<Long, Double> history = resDynamicStats.getHistory();
1666	for (Long key : history.keySet()) {
1667	if(history.get(key) > maxValue){
1668	maxValue = history.get(key) ;
1669	}
1670	}
1671	return maxValue;
1672	}
1673
1674
1675
1676
1677	/*********** TASK STATISTICS SECTION ************/
1678	public void gatherTaskStatistics() {
1679
1680	List<JobInterface<?>> jobs = users.getAllReceivedJobs();
1681	Collections.sort(jobs, new JobIdComparator());
1682
1683	ganttDiagramTaskSeriesCollection = new TaskSeriesCollection();
1684	ganttDiagramWaitingTimeSeriesCollection = new TaskSeriesCollection();
1685
1686	PrintStream taskStatsFile = null;
1687	try {
1688	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX + simulationIdentifier + "_"
1689	+ TASKS_STATISTICS_OUTPUT_FILE_NAME);
1690	taskStatsFile = new PrintStream(new FileOutputStream(file));
1691	} catch (IOException e) {
1692	taskStatsFile = null;
1693	}
1694
1695	PrintStream jobStatsFile = null;
1696	if (configuration.createjobsstatistics) {
1697	try {
1698	File file = new File(outputFolderName + STATS_FILE_NAME_PREFIX + simulationIdentifier + "_"
1699	+ JOBS_STATISTICS_OUTPUT_FILE_NAME);
1700	jobStatsFile = new PrintStream(new FileOutputStream(file));
1701	} catch (IOException e) {
1702	jobStatsFile = null;
1703	}
1704	}
1705
1706	for (int i = 0; i < jobs.size(); i++) {
1707	Job job = (Job) jobs.get(i);
1708
1709	List<ExecTask> execList = jobRegistry.getTasks().getExecutables(job.getId());
1710	List<TaskStats> taskStatsList = new ArrayList<TaskStats>();
1711
1712	this.serializer.setFieldSeparator(SEPARATOR);
1713
1714	for (int j = 0; j < execList.size(); j++) {
1715
1716	Executable task = (Executable) execList.get(j);
1717	if(task.isFinished() == false){
1718	continue;
1719	}
1720	TaskStats taskStats = createTaskStats(task);
1721	if (taskStats != null && taskStatsFile != null) {
1722	Object txt = taskStats.serialize(serializer);
1723	taskStatsFile.println(txt);
1724	}
1725	if (taskStats != null && taskStats.getExecFinishDate() != -1) {
1726	if (configuration.createsimulationstatistics) {
1727	createAccumulatedTaskSimulationStatistic(taskStats);
1728	}
1729	allTasksFinished &= task.isFinished();
1730	if (generateDiagrams) {
1731	if (configuration.creatediagrams_tasks)
1732	createTaskDiagramData(task);
1733	if (configuration.creatediagrams_taskswaitingtime)
1734	createTaskWaitingTimeDiagramData(task);
1735	}
1736	taskStatsList.add(taskStats);
1737	}
1738	}
1739	if (configuration.createjobsstatistics && taskStatsList.size() > 0) {
1740
1741	this.serializer.setFieldSeparator(SEPARATOR);
1742	JobStats jobStats = createJobStats(taskStatsList);
1743	if (jobStatsFile != null) {
1744	Object txt = jobStats.serialize(serializer);
1745	jobStatsFile.println(txt);
1746	}
1747	}
1748	}
1749
1750	if (configuration.createsimulationstatistics) {
1751	accStats.makespan.add(maxCj);
1752	accStats.delayedTasks.add(numOfdelayedTasks);
1753	accStats.failedRequests.add(users.getAllSentTasks().size() - users.getFinishedTasksCount());
1754	if(resourceController.getScheduler().getType().getName().equals("GS")){
1755	for(Scheduler sched: resourceController.getScheduler().getChildren()){
1756	accStats.meanQueueLength.add(getSchedulerQueueStats(sched));
1757	}
1758	}
1759	else {
1760	accStats.meanQueueLength.add(getSchedulerQueueStats(resourceController.getScheduler()));
1761	}
1762	}
1763
1764	saveTaskGanttDiagrams();
1765
1766	if (taskStatsFile != null) {
1767	taskStatsFile.close();
1768	}
1769	if (jobStatsFile != null) {
1770	jobStatsFile.close();
1771	}
1772	}
1773
1774	private double getSchedulerQueueStats(Scheduler scheduler) {
1775
1776	Sim_stat stats = scheduler.get_stat();
1777	List<Object[]> measures = stats.get_measures();
1778	for (Object[] info : measures) {
1779	String measure = (String) info[0];
1780	if (measure
1781	.startsWith(DCWormsConstants.TASKS_QUEUE_LENGTH_MEASURE_NAME)) {
1782	return stats.average(measure);
1783	}
1784	}
1785	return 0;
1786	}
1787
1788	private TaskStats createTaskStats(Executable task) {
1789	TaskStats taskStats = new TaskStats(task, startSimulationTime);
1790	String uniqueTaskID = getUniqueTaskId(task);
1791	taskStats.setProcessors(taskToPEMap.get(uniqueTaskID));
1792	return taskStats;
1793	}
1794
1795	private JobStats createJobStats(List<TaskStats> tasksStats) {
1796
1797	String jobID = ((TaskStats) tasksStats.get(0)).getJobID();
1798	JobStats jobStats = new JobStats(jobID);
1799	double maxCj = 0;
1800
1801	for (int i = 0; i < tasksStats.size(); i++) {
1802
1803	TaskStats task = (TaskStats) tasksStats.get(i);
1804	jobStats.meanTaskStartTime.add(task.getStartTime());
1805	jobStats.meanTaskCompletionTime.add(task.getCompletionTime());
1806	maxCj = Math.max(maxCj, task.getCompletionTime());
1807	jobStats.meanTaskExecutionTime.add(task.getExecutionTime());
1808	jobStats.meanTaskWaitingTime.add(task.getWaitingTime());
1809	jobStats.meanTaskFlowTime.add(task.getFlowTime());
1810	jobStats.meanTaskGQ_WaitingTime.add(task.getGQ_WaitingTime());
1811	jobStats.lateness.add(task.getLateness());
1812	jobStats.tardiness.add(task.getTardiness());
1813	}
1814	jobStats.makespan.add(maxCj);
1815
1816	return jobStats;
1817	}
1818
1819	private void createAccumulatedTaskSimulationStatistic(TaskStats taskStats) {
1820	accStats.meanTaskFlowTime.add(taskStats.getFlowTime());
1821	accStats.meanTaskExecutionTime.add(taskStats.getExecutionTime());
1822	accStats.meanTaskCompletionTime.add(taskStats.getCompletionTime());
1823	accStats.meanTaskWaitingTime.add(taskStats.getWaitingTime());
1824	accStats.meanTaskStartTime.add(taskStats.getStartTime());
1825
1826	accStats.lateness.add(taskStats.getLateness());
1827	accStats.tardiness.add(taskStats.getTardiness());
1828
1829	if (taskStats.getExecFinishDate() == -1) {
1830	numOfNotExecutedTasks++;
1831	}
1832	if (taskStats.getTardiness() > 0.0) {
1833	numOfdelayedTasks++;
1834	}
1835
1836	maxCj = Math.max(maxCj, taskStats.getCompletionTime());
1837	}
1838
1839	private void createTaskDiagramData(Executable task) {
1840	String uniqueTaskID = getUniqueTaskId(task);
1841
1842	String resID = task.getSchedulerName();
1843
1844	long execStartTime = Double.valueOf(task.getExecStartTime()).longValue() * MILLI_SEC;
1845	long execEndTime = Double.valueOf(task.getFinishTime()).longValue() * MILLI_SEC;
1846
1847	TaskSeries taskRes = ganttDiagramTaskSeriesCollection.getSeries(resID);
1848	if (taskRes == null) {
1849	taskRes = new TaskSeries(resID);
1850	ganttDiagramTaskSeriesCollection.add(taskRes);
1851	}
1852	org.jfree.data.gantt.Task ganttTask = new org.jfree.data.gantt.Task(uniqueTaskID, new Date(execStartTime),
1853	new Date(execEndTime));
1854	if (!task.isFinished()) {
1855	ganttTask.setPercentComplete(1.0);
1856	}
1857	taskRes.add(ganttTask);
1858	}
1859
1860	private void createTaskWaitingTimeDiagramData(Executable task) {
1861	String uniqueTaskID = getUniqueTaskId(task);
1862
1863	String resID = task.getSchedulerName();
1864
1865	long execStartTime = Double.valueOf(task.getExecStartTime()).longValue() * MILLI_SEC;
1866	long execEndTime = Double.valueOf(task.getFinishTime()).longValue() * MILLI_SEC;
1867
1868	TaskSeries waitRes = ganttDiagramWaitingTimeSeriesCollection.getSeries(resID);
1869	if (waitRes == null) {
1870	waitRes = new TaskSeries(resID);
1871	ganttDiagramWaitingTimeSeriesCollection.add(waitRes);
1872	}
1873
1874	long sub = Double.valueOf(task.getSubmissionTime()).longValue() * MILLI_SEC;
1875	org.jfree.data.gantt.Task wait_exec = new org.jfree.data.gantt.Task(uniqueTaskID, new Date(sub), new Date(
1876	execEndTime));
1877	org.jfree.data.gantt.Task exec = new org.jfree.data.gantt.Task(uniqueTaskID, new Date(execStartTime), new Date(
1878	execEndTime));
1879	if (!task.isFinished()) {
1880	exec.setPercentComplete(1.0);
1881	}
1882	wait_exec.addSubtask(wait_exec);
1883	wait_exec.addSubtask(exec);
1884	waitRes.add(wait_exec);
1885	}
1886
1887	private boolean saveTaskGanttDiagrams() {
1888
1889	JFreeChart taskDiagram = null;
1890	JFreeChart waitingTimeDiagram = null;
1891
1892	if (!generateDiagrams)
1893	return false;
1894
1895	String fileName = new File(outputFolderName, DIAGRAMS_FILE_NAME_PREFIX + simulationIdentifier + "_")
1896	.getAbsolutePath();
1897
1898	String chartName = "Task diagram for " + DataCenterWorkloadSimulator.SIMULATOR_NAME;
1899	String simulationTime = "Simulation time";
1900	Title subtitle = new TextTitle("created for \"" + simulationIdentifier + "\" at "
1901	+ Calendar.getInstance().getTime().toString());
1902
1903	if (configuration.creatediagrams_tasks) {
1904	taskDiagram = getTaskDiagram(chartName, subtitle, simulationTime);
1905	if (!saveCategoryChart(taskDiagram, fileName + "Tasks", null /* "{0}" */))
1906	return false;
1907	}
1908
1909	chartName = "Task waiting times diagram for " + DataCenterWorkloadSimulator.SIMULATOR_NAME;
1910	if (configuration.creatediagrams_taskswaitingtime) {
1911	waitingTimeDiagram = getTaskWaitingTimeDiagram(chartName, subtitle, simulationTime);
1912	if (!saveCategoryChart(waitingTimeDiagram, fileName + "Waiting_Time", null /* "Task {1} at {0}" */))
1913	return false;
1914	}
1915
1916	return true;
1917	}
1918
1919	private JFreeChart getTaskDiagram(String chartName, Title subtitle, String simulationTime) {
1920	String tasks = "Tasks";
1921	boolean urls = false;
1922	boolean tooltip = true;
1923	boolean legend = true;
1924	JFreeChart chart = ChartFactory.createGanttChart(chartName, tasks, simulationTime,
1925	ganttDiagramTaskSeriesCollection, legend, tooltip, urls);
1926	chart.addSubtitle(subtitle);
1927	GanttRenderer re = (GanttRenderer) chart.getCategoryPlot().getRenderer();
1928	re.setCompletePaint(Color.black);
1929	return chart;
1930	}
1931
1932	private JFreeChart getTaskWaitingTimeDiagram(String chartName, Title subtitle, String simulationTime) {
1933	String tasks = "Tasks";
1934	boolean urls = false;
1935	boolean tooltip = true;
1936	boolean legend = true;
1937	JFreeChart chart = ChartFactory.createGanttChart(chartName, tasks, simulationTime,
1938	ganttDiagramWaitingTimeSeriesCollection, legend, tooltip, urls);
1939	chart.addSubtitle(subtitle);
1940	chart.getPlot().setForegroundAlpha(ALPHA);
1941	GanttRenderer re = (GanttRenderer) chart.getCategoryPlot().getRenderer();
1942	re.setCompletePaint(Color.black);
1943	return chart;
1944	}
1945
1946	private boolean saveCategoryChart(JFreeChart c, String fileName, String labelFormat) {
1947	c.setNotify(false);
1948	c.setAntiAlias(true);
1949	c.setTextAntiAlias(true);
1950	c.setBorderVisible(false);
1951
1952	/*Font fontt=new Font("Arial",Font.BOLD,26);
1953
1954	c.getTitle().setFont(fontt);
1955	Font font=new Font("Arial",0,20);
1956
1957	CategoryPlot p = (CategoryPlot)c.getPlot();
1958	CategoryAxis daxis = p.getDomainAxis();
1959	daxis.setTickLabelFont(font);
1960	daxis.setLabelFont(font);
1961
1962	font=new Font("Arial",0,20);
1963
1964	ValueAxis raxis = p.getRangeAxis();
1965	raxis.setTickLabelFont(font);
1966	raxis.setLabelFont(font);
1967
1968	font=new Font("Arial",0,20);
1969	c.getLegend().setItemFont(font); */
1970
1971	CategoryPlot categoryplot = (CategoryPlot) c.getPlot();
1972	categoryplot.setDomainGridlinesVisible(true);
1973
1974	if (labelFormat != null) {
1975	CategoryItemRenderer rend = categoryplot.getRenderer();
1976	rend.setBaseItemLabelsVisible(true);
1977	ItemLabelPosition position = new ItemLabelPosition(ItemLabelAnchor.CENTER, TextAnchor.CENTER);
1978	rend.setBasePositiveItemLabelPosition(position);
1979	NumberFormat nf = NumberFormat.getInstance();
1980	CategoryItemLabelGenerator gen = new StandardCategoryItemLabelGenerator(labelFormat, nf);
1981	rend.setBaseItemLabelGenerator(gen);
1982	}
1983
1984	int rows = c.getCategoryPlot().getDataset().getColumnKeys().size();
1985
1986	int height = 900;
1987	if (rows > 45) {
1988	height = rows * 20;
1989	}
1990	int width = calculateChartWidth(c);
1991	CategoryItemRenderer rend = c.getCategoryPlot().getRenderer();
1992
1993	for (int i = 0; i < c.getCategoryPlot().getDataset().getRowKeys().size(); i++) {
1994	Paint collor = c.getCategoryPlot().getDrawingSupplier().getNextPaint();
1995	rend.setSeriesOutlinePaint(i, collor);
1996	rend.setSeriesPaint(i, collor);
1997	}
1998
1999	ChartRenderingInfo info = new ChartRenderingInfo();
2000	// info.setEntityCollection(null);
2001	File f = new File(fileName + "." + ImageFormat.PNG);
2002	final String dcwormsSubtitle = "Created by " + DataCenterWorkloadSimulator.SIMULATOR_NAME + " http://www.gssim.org/";
2003	c.addSubtitle(new TextTitle(dcwormsSubtitle));
2004	boolean encodeAlpha = false;
2005	int compression = 9;
2006	try {
2007	ChartUtilities.saveChartAsPNG(f, c, width, height, info, encodeAlpha, compression);
2008
2009	} catch (IOException e) {
2010	e.printStackTrace();
2011	return false;
2012	} catch (Exception e) {
2013	if (log.isErrorEnabled())
2014	log.error("The png file (" + fileName
2015	+ ")\nwill not be generated (It can be too large data size problem)", e);
2016	} catch (Throwable t) {
2017	log.error("The png file (" + fileName + ")\nwill not be generated (It can be too large data size problem)",
2018	t);
2019	}
2020
2021	return true;
2022	}
2023
2024	private int calculateChartWidth(JFreeChart c) {
2025
2026	int ganttWidth = GANTT_WIDTH;
2027	int rows = c.getCategoryPlot().getDataset().getColumnKeys().size();
2028
2029	if (rows > 45) {
2030	int height = rows * 20;
2031	ganttWidth = Math.max((int) (height * 1.5), GANTT_WIDTH);
2032	}
2033	return ganttWidth;
2034	}
2035
2036	private static class JobIdComparator implements Comparator<JobInterface<?>> {
2037
2038	public int compare(JobInterface<?> o1, JobInterface<?> o2) {
2039	Integer o1int;
2040	Integer o2int;
2041	try{
2042	o1int = Integer.parseInt(o1.getId());
2043	o2int = Integer.parseInt(o2.getId());
2044	return o1int.compareTo(o2int);
2045	} catch(NumberFormatException e){
2046	return o1.getId().compareTo(o2.getId());
2047	}
2048	}
2049	}
2050
2051	private static class peIdComparator implements Comparator<ComputingResource> {
2052
2053	public int compare(ComputingResource o1, ComputingResource o2) {
2054	Integer o1int = 0;
2055	Integer o2int = 0;
2056	String o1string = null;
2057	String o2string = null;
2058	try {
2059	if(o1.getFullName().contains("_")){
2060	o1string = o1.getFullName().substring(0, o1.getFullName().lastIndexOf("_"));
2061	o1int = Integer.parseInt(o1.getFullName().substring(o1.getFullName().lastIndexOf("_") + 1));
2062	} else {
2063	o1string = o1.getFullName();
2064	}
2065
2066	if(o2.getFullName().contains("_")){
2067	o2string = o2.getFullName().substring(0, o2.getFullName().lastIndexOf("_"));
2068	o2int = Integer.parseInt(o2.getFullName().substring(o2.getFullName().lastIndexOf("_") + 1));
2069	}else {
2070	o2string = o2.getFullName();
2071	}
2072	} catch (Exception e) {
2073	o1int = 0;
2074	o2int = 0;
2075	o1string = o1.getFullName();
2076	o2string = o2.getFullName();
2077	}
2078
2079
2080	if(o1int.compareTo(o2int) != 0)
2081	return o1int.compareTo(o2int);
2082	else
2083	return o1string.compareTo(o2string);
2084	}
2085	}
2086
2087	public DCwormsAccumulator getStats(String name) {
2088	return statsData.get(name);
2089	}
2090
2091	public boolean accumulatedStats() {
2092	return configuration.createsimulationstatistics;
2093	}
2094
2095	}
2096
2097	class ResStat{
2098
2099	public long getStartDate() {
2100	return startDate;
2101	}
2102	public void setStartDate(long startDate) {
2103	this.startDate = startDate;
2104	}
2105	public long getEndDate() {
2106	return endDate;
2107	}
2108	public void setEndDate(long endDate) {
2109	this.endDate = endDate;
2110	}
2111	public String getTaskID() {
2112	return taskID;
2113	}
2114	public void setTaskID(String taskID) {
2115	this.taskID = taskID;
2116	}
2117
2118	public ResStat(String peName, long startDate, long endDate, String taskID) {
2119	this.startDate = startDate;
2120	this.endDate = endDate;
2121	this.taskID = taskID;
2122	this.peName = peName;
2123	}
2124	public String getPeName() {
2125	return peName;
2126	}
2127	public void setPeName(String peName) {
2128	this.peName = peName;
2129	}
2130	public ResStat(String peName, ResourceType resType, long startDate, long endDate, String taskID) {
2131	this.peName = peName;
2132	this.resType = resType;
2133	this.startDate = startDate;
2134	this.endDate = endDate;
2135	this.taskID = taskID;
2136	}
2137	String peName;
2138	ResourceType resType;
2139	long startDate;
2140	long endDate;
2141	String taskID;
2142
2143	public ResourceType getResType() {
2144	return resType;
2145	}
2146	public void setResType(ResourceType resType) {
2147	this.resType = resType;
2148	}
2149	}
2150
2151	enum Stats{
2152	textLoad,
2153	chartLoad,
2154	textEnergy,
2155	chartEnergy,
2156	textAirflow,
2157	chartAirflow,
2158	textTemperature,
2159	chartTemperature,
2160	textUsefulWork;
2161	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format