Changeset 743

Timestamp:

01/03/13 17:19:06 (12 years ago)

Author:

ariel

Message:

final

Location:

papers/SMPaT-2012_DCWoRMS

Files:

• elsarticle-DCWoRMS.aux (modified) (3 diffs)
• elsarticle-DCWoRMS.pdf (modified) (previous)
• elsarticle-DCWoRMS.tex (modified) (23 diffs)

papers/SMPaT-2012_DCWoRMS/elsarticle-DCWoRMS.aux

@@ -1 +1 @@
 \relax 
 \citation{koomey}
-\citation{hintemann}
 \citation{pue}
+\Newlabel{psnc}{a}
+\Newlabel{put}{b}
 \@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}}
 \citation{games}
 \citation{fit4green}
-\citation{montblanc}
 \citation{networks}
-\citation{sla}
 \citation{sgi}
 \citation{colt}
 \citation{ecocooling}
+\citation{ff}
+\citation{DCD_Romonet}
+\citation{CloudSim}
+\citation{e2dc12}
+\citation{coolemall}
 \citation{GreenCloud}
 \citation{CloudSim}
 \citation{DCSG}
 \@writefile{toc}{\contentsline {section}{\numberline {2}Related Work}{3}}
+\newlabel{sota}{{2}{3}}
 \citation{GSSIM}
 \@writefile{toc}{\contentsline {section}{\numberline {3}DCWoRMS}{5}}
@@ -37 +42 @@
 \@writefile{toc}{\contentsline {paragraph}{\textbf  {Power consumption model}}{10}}
 \@writefile{toc}{\contentsline {paragraph}{\textbf  {Power management interface}}{10}}
-\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.4.2}Air throughput management concept}{11}}
-\@writefile{toc}{\contentsline {paragraph}{\textbf  {Air throughput profile}}{11}}
-\@writefile{toc}{\contentsline {paragraph}{\textbf  {Air throughput model}}{11}}
-\@writefile{toc}{\contentsline {paragraph}{\textbf  {Air throughput management interface}}{11}}
-\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces  Air throughput modeling}}{12}}
-\newlabel{fig:airModel}{{4}{12}}
-\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.4.3}Thermal management concept}{12}}
-\@writefile{toc}{\contentsline {paragraph}{\textbf  {Thermal profile}}{12}}
-\@writefile{toc}{\contentsline {paragraph}{\textbf  {Temperature estimation model}}{12}}
-\@writefile{toc}{\contentsline {paragraph}{\textbf  {Thermal resource management interface}}{12}}
-\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces  Temperature estimation modeling}}{13}}
-\newlabel{fig:tempModel}{{5}{13}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {3.5}Application performance modeling}{13}}
-\newlabel{sec:apps}{{3.5}{13}}
-\@writefile{toc}{\contentsline {section}{\numberline {4}Modeling of energy efficiency in DCWoRMS}{14}}
-\newlabel{eq:E}{{1}{14}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Power consumption models}{15}}
-\newlabel{sec:power}{{4.1}{15}}
-\newlabel{eq:ohm-law}{{2}{15}}
-\newlabel{fig:power}{{4.1}{16}}
-\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces Average power usage with regard to CPU frequency\\ \textbf  {Tests}: Linpack (\emph  {green}), Abinit (\emph  {purple}), Namd (\emph  {blue}) and Cpuburn (\emph  {red}). \\ }}{16}}
-\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces  Power in time for highest frequency}}{16}}
-\newlabel{fig:fans_P}{{7}{16}}
-\newlabel{eq:static}{{3}{17}}
-\newlabel{eq:load}{{4}{17}}
-\newlabel{eq:app}{{5}{17}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Air throughput models}{17}}
-\newlabel{sec:air}{{4.2}{17}}
-\@writefile{lof}{\contentsline {figure}{\numberline {8}{\ignorespaces  Temperature in time for highest frequency}}{18}}
-\newlabel{fig:tempModel}{{8}{18}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Thermal models}{18}}
-\newlabel{sec:thermal}{{4.3}{18}}
-\@writefile{toc}{\contentsline {section}{\numberline {5}Experiments and evaluation}{19}}
-\newlabel{sec:experiments}{{5}{19}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Testbed description}{19}}
-\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces  RECS system configuration}}{19}}
-\newlabel{testBed}{{1}{19}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Evaluated applications}{20}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Models}{20}}
-\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces  $P_{cpubase}$ values in Watts}}{21}}
-\newlabel{nodeBasePowerUsage}{{2}{21}}
-\@writefile{lot}{\contentsline {table}{\numberline {3}{\ignorespaces  $P_{app}$ values in Watts}}{21}}
-\newlabel{appPowerUsage}{{3}{21}}
-\@writefile{lot}{\contentsline {table}{\numberline {4}{\ignorespaces  Power models accuracy in \%}}{21}}
-\newlabel{expPowerModels}{{4}{21}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {5.4}Methodology}{22}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {5.5}Computational analysis}{22}}
-\@writefile{lot}{\contentsline {table}{\numberline {5}{\ignorespaces  Workload characteristics}}{23}}
-\newlabel{workloadCharacteristics}{{5}{23}}
-\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.5.1}Random approach}{24}}
-\@writefile{lof}{\contentsline {figure}{\numberline {9}{\ignorespaces  Random strategy}}{24}}
-\newlabel{fig:70r}{{9}{24}}
-\@writefile{lof}{\contentsline {figure}{\numberline {10}{\ignorespaces  Random + switching off unused nodes strategy}}{25}}
-\newlabel{fig:70rnpm}{{10}{25}}
-\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.5.2}Energy optimization}{25}}
-\@writefile{lot}{\contentsline {table}{\numberline {6}{\ignorespaces  Measured power of testbed nodes in idle state}}{26}}
-\newlabel{idlePower}{{6}{26}}
-\@writefile{lof}{\contentsline {figure}{\numberline {11}{\ignorespaces  Energy usage optimization strategy}}{26}}
-\newlabel{fig:70eo}{{11}{26}}
-\@writefile{lof}{\contentsline {figure}{\numberline {12}{\ignorespaces  Energy usage optimization + switching off unused nodes strategy}}{27}}
-\newlabel{fig:70eonpm}{{12}{27}}
-\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.5.3}Downgrading frequency}{28}}
-\@writefile{lof}{\contentsline {figure}{\numberline {13}{\ignorespaces  Frequency downgrading strategy}}{28}}
-\newlabel{fig:70dfs}{{13}{28}}
-\@writefile{lof}{\contentsline {figure}{\numberline {14}{\ignorespaces  Schedules obtained for Random strategy (left) and Random + lowest frequency strategy (right) for 10\% of system load}}{29}}
-\newlabel{fig:dfsComp}{{14}{29}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {5.6}Discussion}{29}}
-\@writefile{lot}{\contentsline {table}{\numberline {7}{\ignorespaces  Energy usage [kWh] for different level of system load. R - Random, R+NPM - Random + node power management, EO - Energy optimization, EO+NPM - Energy optimization + node power management, R+LF - Random + lowest frequency}}{29}}
-\newlabel{loadEnergy}{{7}{29}}
-\@writefile{lot}{\contentsline {table}{\numberline {8}{\ignorespaces  Makespan [s] for different level of system load. R - Random, R+NPM - Random + node power management, EO - Energy optimization, EO+NPM - Energy optimization + node power management, R+LF - Random + lowest frequency}}{30}}
-\newlabel{loadMakespan}{{8}{30}}
-\@writefile{toc}{\contentsline {section}{\numberline {6}DCWoRMS application/use cases}{30}}
-\newlabel{sec:coolemall}{{6}{30}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {3.5}Application performance modeling}{11}}
+\newlabel{sec:apps}{{3.5}{11}}
+\@writefile{toc}{\contentsline {section}{\numberline {4}Modeling of energy consumption in DCWoRMS}{12}}
+\newlabel{eq:E}{{1}{12}}
+\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Average power usage with regard to CPU frequency - Linpack (\emph  {green}), Abinit (\emph  {purple}), Namd (\emph  {blue}) and Cpuburn (\emph  {red}).  }}{13}}
+\newlabel{fig:power_freq}{{4}{13}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Power consumption models}{13}}
+\newlabel{sec:power}{{4.1}{13}}
+\newlabel{eq:ohm-law}{{2}{13}}
+\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces  Power in time for the highest frequency}}{14}}
+\newlabel{fig:fans_P}{{5}{14}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Static approach}{14}}
+\newlabel{eq:static}{{3}{14}}
+\citation{fit4green_scheduler}
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Resource load}{15}}
+\newlabel{eq:dynamic}{{4}{15}}
+\citation{e2dc12}
+\newlabel{eq:model}{{7}{16}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {4.4}Application specific}{16}}
+\newlabel{eq:app}{{8}{16}}
+\@writefile{toc}{\contentsline {section}{\numberline {5}Experiments and evaluation}{16}}
+\newlabel{sec:experiments}{{5}{16}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Testbed description}{17}}
+\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces  RECS system configuration}}{17}}
+\newlabel{testBed}{{1}{17}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Evaluated applications}{17}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Models}{18}}
+\newlabel{sec:models}{{5.3}{18}}
+\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces  $P_{cpubase}$ values in Watts}}{18}}
+\newlabel{nodeBasePowerUsage}{{2}{18}}
+\@writefile{lot}{\contentsline {table}{\numberline {3}{\ignorespaces  $P_{app}$ values in Watts}}{18}}
+\newlabel{appPowerUsage}{{3}{18}}
+\@writefile{lot}{\contentsline {table}{\numberline {4}{\ignorespaces  Power models error in \%}}{19}}
+\newlabel{expPowerModels}{{4}{19}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.4}Methodology}{19}}
+\@writefile{lot}{\contentsline {table}{\numberline {5}{\ignorespaces  Workload characteristics}}{20}}
+\newlabel{workloadCharacteristics}{{5}{20}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.5}Computational analysis}{20}}
+\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.5.1}Random approach}{21}}
+\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces  Random strategy}}{21}}
+\newlabel{fig:70r}{{6}{21}}
+\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces  Random + switching off unused nodes strategy}}{22}}
+\newlabel{fig:70rnpm}{{7}{22}}
+\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.5.2}Energy optimization}{22}}
+\@writefile{lot}{\contentsline {table}{\numberline {6}{\ignorespaces  Measured power of testbed nodes in idle state}}{23}}
+\newlabel{idlePower}{{6}{23}}
+\@writefile{lof}{\contentsline {figure}{\numberline {8}{\ignorespaces  Energy usage optimization strategy}}{23}}
+\newlabel{fig:70eo}{{8}{23}}
+\@writefile{lof}{\contentsline {figure}{\numberline {9}{\ignorespaces  Energy usage optimization + switching off unused nodes strategy}}{24}}
+\newlabel{fig:70eonpm}{{9}{24}}
+\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.5.3}Downgrading frequency}{25}}
+\@writefile{lof}{\contentsline {figure}{\numberline {10}{\ignorespaces  Frequency downgrading strategy}}{25}}
+\newlabel{fig:70dfs}{{10}{25}}
+\@writefile{lof}{\contentsline {figure}{\numberline {11}{\ignorespaces  Schedules obtained for Random strategy (left) and Random + lowest frequency strategy (right) for 10\% of system load}}{26}}
+\newlabel{fig:dfsComp}{{11}{26}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {5.6}Discussion}{26}}
+\@writefile{lot}{\contentsline {table}{\numberline {7}{\ignorespaces  Energy usage [kWh] for different level of system load. R - Random, R+NPM - Random + node power management, EO - Energy optimization, EO+NPM - Energy optimization + node power management, R+LF - Random + lowest frequency}}{26}}
+\newlabel{loadEnergy}{{7}{26}}
+\@writefile{lot}{\contentsline {table}{\numberline {8}{\ignorespaces  Makespan [s] for different level of system load. R - Random, R+NPM - Random + node power management, EO - Energy optimization, EO+NPM - Energy optimization + node power management, R+LF - Random + lowest frequency}}{27}}
+\newlabel{loadMakespan}{{8}{27}}
+\@writefile{toc}{\contentsline {section}{\numberline {6}Conclusions and future work}{27}}
 \bibcite{fit4green}{{1}{}{{}}{{}}}
-\bibcite{CloudSim}{{2}{}{{}}{{}}}
-\bibcite{DCSG}{{3}{}{{}}{{}}}
-\bibcite{DCD_Romonet}{{4}{}{{}}{{}}}
-\bibcite{networks}{{5}{}{{}}{{}}}
-\bibcite{Ghislain}{{6}{}{{}}{{}}}
-\bibcite{games}{{7}{}{{}}{{}}}
-\@writefile{toc}{\contentsline {section}{\numberline {7}Conclusions and future work}{32}}
-\newlabel{}{{7}{32}}
-\bibcite{GreenCloud}{{8}{}{{}}{{}}}
-\bibcite{sla}{{9}{}{{}}{{}}}
+\newlabel{}{{6}{28}}
+\bibcite{e2dc12}{{2}{}{{}}{{}}}
+\bibcite{CloudSim}{{3}{}{{}}{{}}}
+\bibcite{DCSG}{{4}{}{{}}{{}}}
+\bibcite{DCD_Romonet}{{5}{}{{}}{{}}}
+\bibcite{networks}{{6}{}{{}}{{}}}
+\bibcite{Ghislain}{{7}{}{{}}{{}}}
+\bibcite{games}{{8}{}{{}}{{}}}
+\bibcite{GreenCloud}{{9}{}{{}}{{}}}
 \bibcite{GSSIM}{{10}{}{{}}{{}}}
 \bibcite{GSSIM_Energy}{{11}{}{{}}{{}}}
-\bibcite{hintemann}{{12}{}{{}}{{}}}
-\bibcite{koomey}{{13}{}{{}}{{}}}
+\bibcite{koomey}{{12}{}{{}}{{}}}
+\bibcite{fit4green_scheduler}{{13}{}{{}}{{}}}
 \bibcite{GWF}{{14}{}{{}}{{}}}
 \bibcite{SLURM}{{15}{}{{}}{{}}}
@@ -132 +124 @@
 \bibcite{coolemall}{{19}{}{{}}{{}}}
 \bibcite{ecocooling}{{20}{}{{}}{{}}}
-\bibcite{montblanc}{{21}{}{{}}{{}}}
+\bibcite{ff}{{21}{}{{}}{{}}}
 \bibcite{pue}{{22}{}{{}}{{}}}
 \bibcite{sgi}{{23}{}{{}}{{}}}
-\providecommand\NAT@force@numbers{}\NAT@force@numbers
+\global\NAT@numberstrue

papers/SMPaT-2012_DCWoRMS/elsarticle-DCWoRMS.tex

@@ -104 +104 @@
 %% \fntext[label3]{}
 
-\title{DCWoRMS - a tool for simulation of energy efficiency in Grids and Clouds }
+\title{DCWoRMS - a tool for simulation of energy efficiency in distributed computing infrastructures}
 
 %% use optional labels to link authors explicitly to addresses:
@@ -111 +111 @@
 %% \address[label2]{<address>}
 
-\author{Krzysztof Kurowski, Ariel Oleksiak, Wojciech Piatek, Tomasz Piontek, Andrzej Przybyszewski}
+%\author{Krzysztof Kurowski, Ariel Oleksiak, Wojciech Piatek, Tomasz Piontek, Andrzej Przybyszewski, Jan Węglarz}
+
+
+\author[psnc]{K.~ Kurowski}
+
+%\ead{krzysztof.kurowski@man.poznan.pl}
+
+\author[psnc]{A.~Oleksiak}
+
+%\ead{ariel@man.poznan.pl}
+
+\author[psnc]{W.~Piatek}
+
+%\ead{piatek@man.poznan.pl}
+
+\author[psnc]{T.~Piontek}
+
+\author[psnc]{A.~Przybyszewski}
+
+\author[psnc,put]{J.~Weglarz}
+
+%\cortext[cor1]{Corresponding author}
+
+\address[psnc]{Poznan Supercomputing and Networking Center, Noskowskiego~10, Poznan, Poland}
+
+\address[put]{Institute of Computing Science, Poznan University of Technology,
+\newline
+Piotrowo~2, Poznan, Poland}
+
 
 
@@ -125 +153 @@
 \begin{keyword}
 %% keywords here, in the form: keyword \sep keyword
+simulations, workload and resource modeling, energy-efficiency, data centers
 
 %% MSC codes here, in the form: \MSC code \sep code
@@ -141 +170 @@
 \section{Introduction}
 
-Large-scale computing infrastructures such as grids and clouds caused quick development of data centers. Nowadays, data centers are responsible for around 2\% of the global energy consumption making it equal to the demand of aviation industry \cite{koomey}. In many current data centers the actual IT equipment uses only half of the total energy (e.g. 45-62\% in \cite{hintemann}) while most of the remaining part is required for cooling and air movement resulting in poor Power Usage Effectiveness (PUE) \cite{pue} values. Large energy needs and significant $CO_2$ emissions caused that issues related to cooling, heat transfer, and IT infrastructure location are more and more carefully studied during planning and operation of data centers.
-Even if we take ecological and footprint issues aside, the amount of consumed energy can impose strict limits on data centers. First of all, energy bills may reach millions euros making computations expensive. 
-Furthermore, available power supply is usually limited so it also may reduce data center development capabilities, especially looking at challenges related to exascale computing breakthrough foreseen within this decade.
-
-For these reasons many efforts were undertaken to measure and study energy efficiency of data centers. Some of projects focused on data center monitoring and management \cite{games}\cite{fit4green} whereas others on prototypes of low power computing infrastructures \cite{montblanc}. Studies included aspects such as energy efficiency of networks \cite{networks} and service level agreements related to energy consumption \cite{sla}. Additionally, vendors offer a wide spectrum of energy efficient solutions for computing and cooling \cite{sgi}\cite{colt}\cite{ecocooling}. However, a variety of solutions and configuration options can be applied planning new or upgrading existing data centers.
+Rising popularity of large-scale computing infrastructures such as grids and clouds caused quick development of data centers. Nowadays, data centers are responsible for around 2\% of the global energy consumption making it equal to the demand of aviation industry \cite{koomey}. Moreover, in many current data centers the actual IT equipment uses only half of the total energy whereas most of the remaining part is required for cooling and air movement resulting in poor Power Usage Effectiveness (PUE) \cite{pue} values. Large energy needs and significant $CO_2$ emissions caused that issues related to cooling, heat transfer, and IT infrastructure location are more and more carefully studied during planning and operation of data centers.
+%Even if we take ecological and footprint issues aside, the amount of consumed energy can impose strict limits on data centers. First of all, energy bills may reach millions euros making computations expensive. 
+%Furthermore, available power supply is usually limited so it also may reduce data center development capabilities, especially looking at challenges related to exascale computing breakthrough foreseen within this decade.
+
+For these reasons many efforts were undertaken to measure and study energy efficiency of data centers.There are projects focused on data center monitoring and management \cite{games}\cite{fit4green} whereas others on  energy efficiency of networks \cite{networks}. Additionally, vendors offer a wide spectrum of energy efficient solutions for computing and cooling \cite{sgi}\cite{colt}\cite{ecocooling}. However, a variety of solutions and configuration options can be applied planning new or upgrading existing data centers.
 In order to optimize a design or configuration of data center we need a thorough study using appropriate metrics and tools evaluating how much computation or data processing  can be done within given power and energy budget and how it affects temperatures, heat transfers, and airflows within data center. 
 Therefore, there is a need for simulation tools and models that approach the problem from a perspective of end users and take into account all the factors that are critical to understanding and improving the energy efficiency of data centers, in particular, hardware characteristics, applications, management policies, and cooling.
 These tools should support data center designers and operators by answering questions how specific application types, levels of load, hardware specifications, physical arrangements, cooling technology, etc. impact overall data center energy efficiency. 
-
-In this paper we present a Data Center Workload and Resource Management Simulator (DCWoRMS) which enables modeling and simulation of computing infrastructures to estimate their performance, energy consumption, and energy-efficiency metrics for diverse workloads and management policies.
+There are various tools that allow simulation of computing infrastructures. On one hand they include advanced packages for modeling heat transfer and energy consumption in data centers \cite{ff} or tools concentrating on their financial analysis \cite{DCD_Romonet}. On the other hand, there are simulators focusing on computations such as CloudSim \cite{CloudSim}. The CoolEmAll project aims to integrate these approaches and enable advances analysis of data center efficiency taking into account all these aspects \cite{e2dc12}\cite{coolemall}.
+
+One of the results of the CoolEmAll project is the Data Center Workload and Resource Management Simulator (DCWoRMS) which enables modeling and simulation of computing infrastructures to estimate their performance, energy consumption, and energy-efficiency metrics for diverse workloads and management policies.
 We discuss methods of power usage modeling available in the simulator. To this end, we compare results of simulations to measurements of real servers. 
 To demonstrate DCWoRMS capabilities we evaluate impact of several resource management policies on overall energy-efficiency of specific workloads executed on heterogeneous resources.
@@ -156 +186 @@
 The remaining part of this paper is organized as follows. In Section~2 we give a brief overview of the current state of the art concerning modeling and simulation of distributed systems, such as Grids and Clouds, in terms of energy efficiency. Section~3 discusses the main features of DCWoRMS. In particular, it introduces our approach to workload and resource management, presents the concept of energy efficiency modeling and explains how to incorporate a specific application performance model into simulations. Section~4 discusses energy models adopted within the DCWoRMS. In Section~5 we assess the energy models by comparison of simulation results with real measurements. We also present experiments that were performed using DCWoRMS to show various types of resource and scheduling technics allowing to decrease the total energy consumption of the execution of a set of tasks. In Section~6 we explain how to integrate workload and resource simulations with heat transfer simulations within the CoolEmAll project. Final conclusions and directions for future work are given in Section~7.
 
-\section{Related Work}
+\section{Related Work}\label{sota}
 
 The growing importance of energy-efficiency in information technologies led to significant interest in energy saving methods for computing systems. Nevertheless, studies of impact of resource management policies on energy-efficiency of IT infrastructures require a large effort and are difficult to perform in real distributed environments. To overcome these issues, extensive research has been conducted in the area of modeling and simulation and variety of tools that address the green computing have emerged. The most popular ones are: GreenCloud \cite{GreenCloud}, CloudSim \cite{CloudSim} and DCSG Simulator \cite{DCSG}.
@@ -173 +203 @@
 
 Summarizing, DCWoRMS stands out from other tools due to the flexibility in terms of data center equipment and structure definition.
-Moreover, it allows to associate the energy consumption not only with the current power state and resource utilization but also with the particular set of applications running on it. Moreover, it does not limit the user in defining various types of resource management polices. The main strength of CloudSim lies in implementation of the complex scheduling and task execution schemes involving resource virtualization techniques. However,the energy efficiency aspect is limited only to the VM management, The GreenCloud focuses on data center resources with particular attention to the network infrastructure and the most popular energy management approaches. DCSG simulator allows to take into account also non-computing devices, nevertheless it seems to be hardly customizable tool.
+Moreover, it allows to associate the energy consumption not only with the current power state and resource utilization but also with the particular set of applications running on it. Moreover, it does not limit the user in defining various types of resource management polices. The main strength of CloudSim lies in implementation of the complex scheduling and task execution schemes involving resource virtualization techniques. However,the energy efficiency aspect is limited only to the VM management, The GreenCloud focuses on data center resources with particular attention to the network infrastructure and the most popular energy management approaches. DCSG simulator allows to take into account also non-computing devices, nevertheless it seems to be hardly customizable to specific workloads and management policies.
 
 \section{DCWoRMS}
@@ -252 +282 @@
 Presence of detailed resource usage information, current resource energy state description and functional energy management interface enables an implementation of energy-aware scheduling algorithms. Resource energy consumption becomes in this context an additional criterion in the scheduling process, which use various techniques to decrease energy consumption, e.g. workload consolidation, moving tasks between resources to reduce a number of running resources, dynamic power management, cutting down CPU frequency, and others.
 
-\subsubsection{Air throughput management concept}
-
-The presence of an air throughput concept addresses the issue of resource air-cooling facilities provisioning. Using the air throughput profiles and models allows anticipating the air flow level on output of the computing system component, resulting from air-cooling equipment management.
-
-\paragraph{\textbf{Air throughput profile}}
-The air throughput profile, analogously to the power profile, allows specifying supported air flow states. Each air throughput state definition consists of an air flow value and a corresponding power draw. It can represent, for instance, a fan working state. In this way, associating the air throughput profile with the given computing resource, it is possible to describe mounted air-cooling devices.
-Possibility of introducing additional parameters makes the air throughput description extensible for new specific characteristics.
-
-\paragraph{\textbf{Air throughput model}}
-Similar to energy consumption models, the user is provided with a dedicated interface that allows him to describe the resulting air throughput of the computing system components like cabinets or server fans. The general idea of the air throughput modeling is shown in Figure~\ref{fig:airModel}. Accordingly, air flow estimations are based on detailed information about the involved resources, including their air throughput states. 
-
-\begin{figure}[tbp]
-\centering
-\includegraphics[width = 8cm]{fig/airModel.png}
-\caption{\label{fig:airModel} Air throughput modeling}
-\end{figure}
-
-\paragraph{\textbf{Air throughput management interface}}
-The DCWoRMS delivers interfaces that provide access to the air throughput profile data, allows acquiring detailed information concerning current air flow conditions and changes in air flow states. The availability of these interfaces support evaluation of different cooling strategies.
-
-
-
-\subsubsection{Thermal management concept}
-
-The primary motivation behind the incorporation of thermal aspects in the DCWoRMS is to exceed the commonly adopted energy use-cases and apply more sophisticated scenarios. By the means of dedicated profiles and interfaces, it is possible to perform experimental studies involving temperature-aware workload placement.
-
-\paragraph{\textbf{Thermal profile}}
-Thermal profile expresses the thermal specification of resources. It consists of the definition of the thermal design power (TDP), thermal resistance and thermal states that describe how the temperature depends on dissipated heat. For the purposes of more complex experiments, introducing of new, user-defined characteristics is supported. The aforementioned values may be provided for all computing system components distinguishing them, for instance, according to their material parameters and/or models.
-
-\paragraph{\textbf{Temperature estimation model}}
-Thermal profile, complemented with the temperature measurement model implementation may introduce temperature sensors simulation. In this way, users have means to approximately predict the temperature of the simulated objects by taking into account basic thermal characteristics as well as the estimated impact of cooling devices. However, the proposed approach assumes some simplifications that ignore heating and cooling dynamics understood as a heat flow process.
-
-Figure~\ref{fig:tempModel} summarizes relation between model and profile and input data.
-
-\begin{figure}[tbp]
-\centering
-\includegraphics[width = 8cm]{fig/tempModel.png}
-\caption{\label{fig:tempModel} Temperature estimation modeling}
-\end{figure}
-
-\paragraph{\textbf{Thermal resource management interface}}
-As the temperature is highly dependent on the dissipated heat and cooling capacity, thermal resource management is performed via a power and air throughput interface. Nevertheless, the interface provides access to the thermal resource characteristics and the current temperature values
+%\subsubsection{Air throughput management concept}
+
+%The presence of an air throughput concept addresses the issue of resource air-cooling facilities provisioning. Using the air throughput profiles and models allows anticipating the air flow level on output of the computing system component, resulting from air-cooling equipment management.
+
+%\paragraph{\textbf{Air throughput profile}}
+%The air throughput profile, analogously to the power profile, allows specifying supported air flow states. Each air throughput state definition consists of an air flow value and a corresponding power draw. It can represent, for instance, a fan working state. In this way, associating the air throughput profile with the given computing resource, it is possible to describe mounted air-cooling devices.
+%Possibility of introducing additional parameters makes the air throughput description extensible for new specific characteristics.
+
+%\paragraph{\textbf{Air throughput model}}
+%Similar to energy consumption models, the user is provided with a dedicated interface that allows him to describe the resulting air throughput of the computing system components like cabinets or server fans. The general idea of the air throughput modeling is shown in Figure~\ref{fig:airModel}. Accordingly, air flow estimations are based on detailed information about the involved resources, including their air throughput states. 
+
+%\begin{figure}[tbp]
+%\centering
+%\includegraphics[width = 8cm]{fig/airModel.png}
+%\caption{\label{fig:airModel} Air throughput modeling}
+%\end{figure}
+
+%\paragraph{\textbf{Air throughput management interface}}
+%The DCWoRMS delivers interfaces that provide access to the air throughput profile data, allows acquiring detailed information concerning current air flow conditions and changes in air flow states. The availability of these interfaces support evaluation of different cooling strategies.
+
+
+
+%\subsubsection{Thermal management concept}
+
+%The primary motivation behind the incorporation of thermal aspects in the DCWoRMS is to exceed the commonly adopted energy use-cases and apply more sophisticated scenarios. By the means of dedicated profiles and interfaces, it is possible to perform experimental studies involving temperature-aware workload placement.
+
+%\paragraph{\textbf{Thermal profile}}
+%Thermal profile expresses the thermal specification of resources. It consists of the definition of the thermal design power (TDP), thermal resistance and thermal states that describe how the temperature depends on dissipated heat. For the purposes of more complex experiments, introducing of new, user-defined characteristics is supported. The aforementioned values may be provided for all computing system components distinguishing them, for instance, according to their material parameters and/or models.
+
+%\paragraph{\textbf{Temperature estimation model}}
+%Thermal profile, complemented with the temperature measurement model implementation may introduce temperature sensors simulation. In this way, users have means to approximately predict the temperature of the simulated objects by taking into account basic thermal characteristics as well as the estimated impact of cooling devices. However, the proposed approach assumes some simplifications that ignore heating and cooling dynamics understood as a heat flow process.
+
+%Figure~\ref{fig:tempModel} summarizes relation between model and profile and input data.
+
+%\begin{figure}[tbp]
+%\centering
+%\includegraphics[width = 8cm]{fig/tempModel.png}
+%\caption{\label{fig:tempModel} Temperature estimation modeling}
+%\end{figure}
+
+%\paragraph{\textbf{Thermal resource management interface}}
+%As the temperature is highly dependent on the dissipated heat and cooling capacity, thermal resource management is performed via a power and air throughput interface. Nevertheless, the interface provides access to the thermal resource characteristics and the current temperature values
 
 
@@ -311 +341 @@
 
 
-\section{Modeling of energy efficiency in DCWoRMS}
+\section{Modeling of energy consumption in DCWoRMS}
 
 DCWoRMS is an open framework in which various models and algorithms can be investigated as presented in Section \ref{sec:apps}. We discuss possible approaches to modeling that can be applied to simulation of energy-efficiency of distributed computing systems in this section. Additionally, to facilitate the simulation process, DCWoRMS provides some basic implementation of power consumption, air throughput and thermal models. We described them as examples and validate part of them by experiments in real computing system (in Section \ref{sec:experiments}).
@@ -324 +354 @@
 Power function may depend on load and states of resources or even specific applications as explained in \ref{sec:power}. Total energy can be also completed by adding constant power usage of components that does not depend on load or state of resources. 
 
-In large computing systems which are often characterized by high computational density, total energy consumption of computing nodes is not the only result interesting for researchers. Temperature distribution is getting more and more important as it affects energy consumption of cooling devices, which can reach even half of a total data center energy use. In order to obtain accurate values of temperatures heat transfer simulations based on the Computational Fluid Dynamics (CFD) methods  have to be performed. These methods require as an input (i.e. boundary conditions) a heat dissipated by IT hardware and air throughput generated by fans at servers' outlets. Another approach is based on simplified thermal models that without costly CFD calculations provide rough estimations of temperatures. DCWoRMS enables the use of either approaches. In the the former, the output of simulations including power usage of computing nodes in time and air throughput at node outlets can be passed to CFD solver. This option is further elaborated in Section \ref{sec:coolemall}. Simplified thermal models required by the latter approach are proposed in \ref{sec:thermal}.
+In large computing systems which are often characterized by high computational density, total energy consumption of computing nodes is not the only result interesting for researchers. Temperature distribution is getting more and more important as it affects energy consumption of cooling devices, which can reach even half of a total data center energy use. In order to obtain accurate values of temperatures heat transfer simulations based on the Computational Fluid Dynamics (CFD) methods  have to be performed. These methods require as an input (i.e. boundary conditions) a heat dissipated by IT hardware and air throughput generated by fans at servers' outlets. Another approach is based on simplified thermal models that without costly CFD calculations provide rough estimations of temperatures. DCWoRMS enables the use of either approaches. In the the former, the output of simulations including power usage of computing nodes in time and air throughput at node outlets can be passed to CFD solver. 
+%This option is further elaborated in Section \ref{sec:coolemall}. Simplified thermal models required by the latter approach are proposed in \ref{sec:thermal}.
 
 
@@ -331 +362 @@
 As stated above power usage of computing nodes depend on a number of factors. 
 
-Generally, the power consumption of a modern CPU is given by the Ohm's law: 
+%
+\begin{figure}
+\centering
+\includegraphics[width=6cm]{fig/power_default}
+\caption{Average power usage with regard to CPU frequency
+- Linpack (\emph{green}), Abinit (\emph{purple}), Namd (\emph{blue}) and Cpuburn (\emph{red}). \label{fig:power_freq}
+}
+%
+\end{figure}
+
+Generally, the power consumption of a modern CPU is given by the formula: 
 %\cite{intel_speedstep}:
 
@@ -344 +385 @@
 also leads to the reduction of $V_{core}$ and thus the power savings
 from the $P\sim V_{core}^{2}$ relation outweigh the increased computation
-time. However, experiments performed on several HPC servers shown that this dependency does not reflect theoretical shape and is often close to linear as presented in Figure \ref{fig:power}. This phenomenon can be explained by impact of other component than CPU and narrow range of available voltages. A good example of impact by other components is power usage of servers with visible influence of fans as illustrated in Figure \ref{fig:fans_P}.
-
-%
-\begin{figure}\label{fig:power}
-\centering
-\includegraphics[width=6cm]{fig/power_default}
-\caption{Average power usage with regard to CPU frequency\protect \\
-\textbf{Tests}: Linpack (\emph{green}), Abinit (\emph{purple}),
-Namd (\emph{blue}) and Cpuburn (\emph{red}). \protect \\
-}
-%
-\end{figure}
+time. However, experiments performed on several HPC servers shown that this dependency does not reflect theoretical shape and is often close to linear as presented in Figure \ref{fig:power_freq}. This phenomenon can be explained by impact of other component than CPU and narrow range of available voltages. A good example of impact by other components is power usage of servers with visible influence of fans as illustrated in Figure \ref{fig:fans_P}.
+
+For these reasons, DCWoRMS allows users to define dependencies between power usage and resource states (such as CPU frequency) in the form of tables or arbitrary functions using energy estimation plugins.
+
+The energy consumption models provided by default can be classified into the following groups, starting from the simplest model up to the more complex ones. Users can easily switch between the given models and incorporate new, visionary scenarios.
 
 \begin{figure}[tbp]
 \centering
-\includegraphics[width = 8cm]{fig/power-fans.png}
-\caption{\label{fig:fans_P} Power in time for highest frequency}
-\end{figure}
-
-
-For these reasons, DCWoRMS allows users to define dependencies between power usage and resource states (such as CPU frequency) in the form of tables or arbitrary functions using energy estimation plugins.
-
-The energy consumption models provided by default can be classified into the following groups, starting from the simplest model up to the more complex ones. Users can easily switch between the given models and incorporate new, visionary scenarios.
-
-\textbf{Static approach} is based on a static definition of resource power usage. This model calculates the total amount of energy consumed by the computing resource system as a sum of energy, consumed by all its components (processors, disks, power adapters, etc.). More advanced versions of this approach assume definition of resource states along with corresponding power usage. This model follows changes of resource power states and sums up the amounts of energy defined for each state.
+\includegraphics[width = 6cm]{fig/power-fans.png}
+\caption{\label{fig:fans_P} Power in time for the highest frequency}
+\end{figure}
+
+
+\subsection{Static approach} 
+Static approach is based on a static definition of resource power usage. This model calculates the total amount of energy consumed by the computing resource system as a sum of energy, consumed by all its components (processors, disks, power adapters, etc.). More advanced versions of this approach assume definition of resource states along with corresponding power usage. This model follows changes of resource power states and sums up the amounts of energy defined for each state.
 In this case, specific values of power usage are defined for all discrete $n$ states as shown in (\ref{eq:static}): 
 
@@ -375 +406 @@
 \end{equation}
 
-\textbf{Resource load} model extends the static power state description and enhances it with real-time resource usage, most often simply the processor load. In this way it enables a dynamic estimation of power usage based on resource basic power usage and state (defined by the static resource description) as well as resource load. For instance, it allows distinguishing between the amount of energy used by idle processors and processors at full load. In this manner, energy consumption is directly connected with power state and describes average power usage by the resource working in a current state.
-In this case, specific values of power usage are defined for all pairs state and load values (discretized to $l$ values) as shown in (\ref{eq:load}): 
+Within DCWoRMS we built in a static approach model that uses common resource states that affect power usage which are the CPU power states. Hence, with each node power state, understood as a possible operating state (p-state), we associated a power consumption value that derives from the averaged values of measurements obtained for different types of application. We distinguish also an idle state. Therefore, the current power usage of the node, can be expressed as: $P = P_{idle} + P_{f}$ where $P$ denotes power consumed by the node, $P_{idle}$ is a power usage of node in idle state and $P_{f}$ stands for power usage of CPU operating at the given frequency level. Additionally, node power states are taken into account to reflect no (or limited) power usage when a node is off.
+
+\subsection{Resource load} 
+Resource load model extends the static power state description and enhances it with real-time resource usage, most often simply the processor load. In this way it enables a dynamic estimation of power usage based on resource basic power usage and state (defined by the static resource description) as well as resource load. For instance, it allows distinguishing between the amount of energy used by idle processors and processors at full load. In this manner, energy consumption is directly connected with power state and describes average power usage by the resource working in a current state.
+In this case, specific values of power usage are defined for all pairs state and load values (discretized to $l$ values) as shown in (\ref{eq:dynamic}): 
 
 \begin{equation}
-(S_1, L_1) \to P_{11}, (S_1, L_2) \to P_{12}, ...,  (S_2, L_1) \to P_{21}, ..., (S_n, L_l) \to P_{nl},  \label{eq:load}
+(S_1, L_1) \to P_{11}, (S_1, L_2) \to P_{12}, ...,  (S_2, L_1) \to P_{21}, ..., (S_n, L_l) \to P_{nl},  \label{eq:dynamic}
 \end{equation}
 
-\textbf{Application specific} model allows expressing differences in the amount of energy required for executing various types of applications at diverse computing resources. It considers all defined system elements (processors, memory, disk, etc.), which are significant in total energy consumption. Moreover, it also assumes that each of these components can be utilized in a different way during the experiment and thus have different impact on total energy consumption. To this end, specific characteristics of resources and applications are taken into consideration. Various approaches are possible including making the estimated power usage dependent on defined classes of applications, ratio between CPU-bound and IO-bound operations, etc.
+A typical functional model of power usage can be based on theoretical dependencies between power and parameters such as CPU frequency, voltage, load, memory usage, etc. In this case CPU power usage for core $i$, $P_i$ can be given according to \ref{eq:ohm-law}. Then, the total CPU power can be calculated as a sum of utilized cores: 
+
+\begin{equation}
+P_{CPU}=P_{idle} + \sum_i^{num\_cores}{P_i}
+\end{equation}
+
+and the whole node power usage as a sum of CPU, memory and other components (e.g., as defined in \cite{fit4green_scheduler}):
+
+\begin{equation}
+P=\sum_i^{num\_cpus}{P_{CPU_i}} + P_{RAM} + P_{HD} + P_{fan} + P_{PSU}
+\end{equation}
+
+Unfortunately, to verify this model and adjust it to the specific hardware, power usage of particular subcomponents such as CPU or memory must be measured. As this is usually difficult, other models, based on a total power use, can be applied.
+
+An example is a model applied in DCWoRMS based on the real measurements (see Section \ref{sec:models} for more details): 
+
+\begin{equation}
+P = P_{idle} + L*P_{cpubase}*c^{(f-f_{base})/100} + P_{app}, \label{eq:model}
+\end{equation}
+
+where $P$ denotes power consumed by the node executing the given application, $P_{idle}$ is a power usage of node in idle state, $L$ is the current utilization level of the node, $P_{cpubase}$ stands for power usage of fully loaded CPU working in the lowest frequency, $c$ is the constant factor indicating the increase of power consumption with respect to the frequency increase $f$ is a current frequency, $f_{base}$ is the lowest available frequency within the given CPU and $P_{app}$ denotes the additional power usage derived from executing a particular application).
+
+
+\subsection{Application specific} 
+Application specific model allows expressing differences in the amount of energy required for executing various types of applications at diverse computing resources. It considers all defined system elements (processors, memory, disk, etc.), which are significant in total energy consumption. Moreover, it also assumes that each of these components can be utilized in a different way during the experiment and thus have different impact on total energy consumption. To this end, specific characteristics of resources and applications are taken into consideration. Various approaches are possible including making the estimated power usage dependent on defined classes of applications, ratio between CPU-bound and IO-bound operations, etc.
 In this case, power usage is an arbitrary function of state, load, and application characteristics as shown in (\ref{eq:app}): 
 
@@ -389 +447 @@
 \end{equation}
 
-\subsection{Air throughput models}\label{sec:air}
-
-The DCWoRMS comes with the following air throughput models.
-By default, air throughput estimations are performed according to the first one.
-
-\textbf{Static} model refers to a static definition of air throughput states. According to this approach, output air flow depends only on the present air cooling working state and the corresponding air throughput value. Each state change triggers the calculations and updates the current air throughput value. This strategy requires only a basic air throughput profile definition.
-
-\textbf{Space} model allows taking into account a duct associated with the investigated air flow. On the basis of the given fan rotation speed and the obstacles before/behind the fans, the output air throughput can be roughly estimated. To this end, additional manufacturer's specifications will be required, including resulting air velocity values and fan duct dimensions. Thus, it is possible to estimate the air flow level not only referring to the current fan operating state but also with respect to the resource and its subcomponent placement. More advanced scenario may consider mutual impact of several air flows.
-
-\subsection{Thermal models}\label{sec:thermal}
-
-\begin{figure}[tbp]
-\centering
-\includegraphics[width = 8cm]{fig/temp-fans.png}
-\caption{\label{fig:tempModel} Temperature in time for highest frequency}
-\end{figure}
-
-
-The following models are supported natively. By default, the static strategy is applied.
-
-\textbf{Static} approach follows the changes in heat, generated by the computing system components and matches the corresponding temperature according to the specified profile. Since it tracks the power consumption variations, corresponding values must be delivered, either from power consumption model or on the basis of user data. Replacing the appropriate temperature values with function based on the defined material properties and/o experimentally measured values can easily extend this model.
-
-\textbf{Ambient} model allows taking into account the surrounding cooling infrastructure. It calculates the device temperature as a function adopted from the static approach and extends it with the influence of cooling method. The efficiency of cooling system may be derived from the current air throughput value.
+%\subsection{Air throughput models}\label{sec:air}
+
+%The DCWoRMS comes with the following air throughput models.
+%By default, air throughput estimations are performed according to the first one.
+
+%\textbf{Static} model refers to a static definition of air throughput states. According to this approach, output air flow depends only on the present air cooling working state and the corresponding air throughput value. Each state change triggers the calculations and updates the current air throughput value. This strategy requires only a basic air throughput profile definition.
+
+%\textbf{Space} model allows taking into account a duct associated with the investigated air flow. On the basis of the given fan rotation speed and the obstacles before/behind the fans, the output air throughput can be roughly estimated. To this end, additional manufacturer's specifications will be required, including resulting air velocity values and fan duct dimensions. Thus, it is possible to estimate the air flow level not only referring to the current fan operating state but also with respect to the resource and its subcomponent placement. More advanced scenario may consider mutual impact of several air flows.
+
+%\subsection{Thermal models}\label{sec:thermal}
+
+%\begin{figure}[tbp]
+%\centering
+%\includegraphics[width = 8cm]{fig/temp-fans.png}
+%\caption{\label{fig:tempModel} Temperature in time for highest frequency}
+%\end{figure}
+
+%The following models are supported natively. By default, the static strategy is applied.
+
+%\textbf{Static} approach follows the changes in heat, generated by the computing system components and matches the corresponding temperature according to the specified profile. Since it tracks the power consumption variations, corresponding values must be delivered, either from power consumption model or on the basis of user data. Replacing the appropriate temperature values with function based on the defined material properties and/o experimentally measured values can easily extend this model.
+
+%\textbf{Ambient} model allows taking into account the surrounding cooling infrastructure. It calculates the device temperature as a function adopted from the static approach and extends it with the influence of cooling method. The efficiency of cooling system may be derived from the current air throughput value.
 
 \section{Experiments and evaluation}\label{sec:experiments}
@@ -419 +476 @@
 \subsection{Testbed description}
 
-To obtain values of power consumption that could be later used in DCWoRMS environment to build the model and to evaluate resource management policies we ran a set of applications / benchmarks on the physical testbed. For experimental purposes we choose the high-density Resource Efficient Cluster Server (RECS) system. The single RECS unit consists of 18 single CPU modules, each of them can be treated as an individual node of PC class. Configuration of our RECS unit is presented in Table~\ref{testBed}. 
+To obtain values of power consumption that could be later used in DCWoRMS environment to build the model and to evaluate resource management policies we ran a set of applications / benchmarks on the physical testbed. For experimental purposes we choose the Christmann high-density Resource Efficient Cluster Server (RECS) system \cite{e2dc12}. The single RECS unit consists of 18 single CPU modules, each of them can be treated as an individual node of PC class. Configuration of our RECS unit is presented in Table~\ref{testBed}. 
 
 \begin {table}[h!]
@@ -460 +517 @@
 
 
-\subsection{Models}
+\subsection{Models}\label{sec:models}
 
 Based on the measured values we evaluated three types of models that can be applied, among others, to the simulation environment.
 
 \textbf{Static}
-This model refers to the static approach presented in Section~\ref{sec:power}. According to the measured values we created a resource power consumption model that is based on a static definition of resource power usage. With each node power state, understood as a possible operating state (p-state), we associated a power consumption value that derives from the averaged values of measurements obtained for different types of application. Therefore, the current power usage of the node, can be expressed as: $P = P_{idle} + P_{f}$ where $P$ denotes power consumed by the node, $P_{idle}$ is a power usage of node in idle state and $P_{f}$ stands for power usage of CPU operating at the given frequency level.
+This model refers to the static approach presented in Section~\ref{sec:power}. According to the measured values we created a resource power consumption model that is based on a static definition of resource power usage. 
+%With each node power state, understood as a possible operating state (p-state), we associated a power consumption value that derives from the averaged values of measurements obtained for different types of application. Therefore, the current power usage of the node, can be expressed as: $P = P_{idle} + P_{f}$ where $P$ denotes power consumed by the node, $P_{idle}$ is a power usage of node in idle state and $P_{f}$ stands for power usage of CPU operating at the given frequency level.
 
 \textbf{Dynamic}
-This model is combination of Resource load and Application specific approaches presented in Section~\ref{sec:power}. Based on the measured values and referring to the existing models presented in literature, we assumed the following equation: $P = P_{idle} + load*P_{cpubase}*c^{(f-f_{base})/100} + P_{app}$, where $P$ denotes power consumed by the node executing the given application, $P_{idle}$ is a power usage of node in idle state, load is the current utilization level of the node, $P_{cpubase}$ stands for power usage of fully loaded CPU working in the lowest frequency, $c$ is the constant factor indicating the increase of power consumption with respect to the frequency increase $f$- is a current frequency, $f_{base}$- is the lowest available frequency within the given CPU and $P_{app}$ denotes the additional power usage derived from executing a particular application).
+This model refers to the Resource load approach presented in Section~\ref{sec:power}. Based on the measured values of the total node power usage for various levels of load and frequencies of CPUs node power usage was defined as in \ref{eq:model}.
+
+%and referring to the existing models presented in literature, we assumed the following equation: $P = P_{idle} + load*P_{cpubase}*c^{(f-f_{base})/100} + P_{app}$, where $P$ denotes power consumed by the node executing the given application, $P_{idle}$ is a power usage of node in idle state, load is the current utilization level of the node, $P_{cpubase}$ stands for power usage of fully loaded CPU working in the lowest frequency, $c$ is the constant factor indicating the increase of power consumption with respect to the frequency increase $f$- is a current frequency, $f_{base}$- is the lowest available frequency within the given CPU and $P_{app}$ denotes the additional power usage derived from executing a particular application).
 
 
@@ -506 +566 @@
 
 \textbf{Mapping}
-In this model we applied the measured values exactly to the power model. Obviously this model is contaminated only with the inaccuracy of the measurements.
+This model refers to the Application specific approach presented in Section~\ref{sec:power}. However, in this model we applied the measured values for each application exactly to the power model. Neither dependencies with load nor application profiles are modeled. Obviously this model is contaminated only with the inaccuracy of the measurements and variability of power usage (caused by other unmeasured factors).
         
 The following table (Table~\ref{expPowerModels}) contains the relative errors of the models with respect to the measured values
@@ -518 +578 @@
 \hline
 \end{tabular}
-\caption {\label{expPowerModels} Power models accuracy in \%}
+\caption {\label{expPowerModels} Power models error in \%}
 \end {table}
+
+Obviously, 0\% error in the case of the Mapping model is caused by the use of a tabular data, which for each application stores a specific power usage. Nevertheless, in all models we face possible deviations from the average caused by power usage fluctuations not explained by variables used in models. These deviations reached around 7\% for each case.
 
 For the experimental purposes we decided to use the latter model. Thus, we introduce into the simulation environment exact values obtained within our testbed, to build both the power profiles of applications as well as the application performance models, denoting the their execution times.
@@ -609 +671 @@
 \end {table}
 
-As mentioned, we assign tasks to nodes minimizing the value of expression: $(P-Pidle)*exec\_time$, where $P$ denotes observed power of the node running the particular application and $exec\_time$ refers to the measured application running time. Based on the application and hardware profiles, we expected that Atom D510 would be the preferred node. Obtained schedule, that is presented in the Gantt chart in Figure~\ref{fig:70eo} along with the energy and system usage, confirmed our assumptions. Atom D510 nodes are nearly fully loaded, while the least energy-favourable AMD nodes are used only when other ones are busy.
+As mentioned, we assign tasks to nodes minimizing the value of expression: $(P-P_{idle})*exec\_time$, where $P$ denotes observed power of the node running the particular application and $exec\_time$ refers to the measured application running time. Based on the application and hardware profiles, we expected that Atom D510 would be the preferred node. Obtained schedule, that is presented in the Gantt chart in Figure~\ref{fig:70eo} along with the energy and system usage, confirmed our assumptions. Atom D510 nodes are nearly fully loaded, while the least energy-favourable AMD nodes are used only when other ones are busy.
 
 \begin{figure}[h!]
@@ -689 +751 @@
 
 Referring to the Table~\ref{loadEnergy}, one should easily note that gain from switching off unused nodes decreases with the increasing workload density. In general, for the highly loaded system such policy does not find an application due to the cost related to this process and relatively small benefits. Another interesting conclusion, reefers to the poor result for Random strategy with downgrading the frequency approach. The lack of improvement on the energy usage criterion for higher system load can be explained by the relatively small or no benefit obtained for prolonging the task execution, and thus, maintaining the node in working state. The cost of longer workload completion, can not be compensate by the very little energy savings derived from the lower operating state of node.
-
-
-
-\section{DCWoRMS application/use cases}\label{sec:coolemall}
-
-DCWoRMS in CoolEmAll, integration with CFD
-
-...
-
-Being based on the GSSIM framework, that has been successfully applied in a substantial number of research projects and academic studies, DCWoRMS with its sophisticated energy extension has become an essential tool for studies of energy efficiency in distributed environments. For this reason, it has been adopted within the CoolEmAll project as a component of Simulation, Visualisation and Decision Support (SVD) Toolkit. In general the main goal of CoolEmAll is to provide advanced simulation, visualisation and decision support tools along with blueprints of computing building blocks for modular data centre environments. Once developed, these tools and blueprints should help to minimise the energy consumption, and consequently the CO2 emissions of the whole IT infrastructure with related facilities. The SVD Toolkit is designed to support the analysis and optimization of IT modern infrastructures. For the recent years the special attention has been paid for energy utilized by the data centers which considerable contributes to the data center operational costs.  Actual power usage and effectiveness of energy saving methods heavily depends on available resources, types of applications and workload properties. Therefore, intelligent resource management policies are gaining popularity when considering the energy efficiency of IT infrastructures.
-Hence, SVD Toolkit integrates also workload management and scheduling policies to support complex modeling and optimization of modern data centres.
-
-The main aim of DCWoRMS within CoolEmAll project is to enable studies of dynamic states of IT infrastructures, like power consumption and air throughput distribution, on the basis of changing workloads, resource model and energy-aware resource management policies.
-In this context, DCWoRMS takes into account the specific workload and application characteristics as well as detailed resource parameters. It will benefit from the CoolEmAll benchmarks and classification of applications and workloads. In particular various types of workload, including data centre workloads using virtualization and HPC applications, may be considered. The knowledge concerning their performance and properties as well as information about their energy consumption and heat production will be used in simulations to study their impact on thermal issues and energy efficiency. Detailed resource characteristics, will be also provided according to the CoolEmAll blueprints. Based on this data, workload simulation will support evaluation process of various resource management approaches. These policies may include a wide spectrum of energy-aware strategies such as workload consolidation/migration, dynamic switching off nodes, DVFS and thermal-aware methods. In addition to typical approaches minimizing energy consumption, policies that prevent too high temperatures in the presence of limited cooling (or no cooling) may also be analyzed. Moreover, apart from the set of predefined strategies, new approaches can easily be applied and examined.
-The outcome of the workload and resource management simulation phase is a distribution of power usage and air throughput for the computing models specified within the SVD Toolkit. These statistics may be analyzed directly by data centre designers and administrators and/or provided as an input to the CFD simulation phase. The former case allows studying how the above metrics change over time, while the latter harness CFD simulations to identify temperature differences between the computing modules, called hot spots. The goal of this scenario is to visualise the behavior of the temperature distribution within a server room with a number of racks for different types of executed workloads and for various policies used to manage these workloads.
+We also demonstrated differences between power usage models. They span from rough static approach to accurate application specific models. However, the latter can be difficult or even infeasible to use as it requires real measurements for specific applications beforehand. This issue can be partially resolved by introducing application profiles and classification, which can deteriorate the accuracy though. This issue is begin studied more deeply within CoolEmAll project. 
+
+
+
+%\section{DCWoRMS application/use cases}\label{sec:coolemall}
+
+%DCWoRMS in CoolEmAll, integration with CFD
+
+%...
+
+%Being based on the GSSIM framework, that has been successfully applied in a substantial number of research projects and academic studies, DCWoRMS with its sophisticated energy extension has become an essential tool for studies of energy efficiency in distributed environments. For this reason, it has been adopted within the CoolEmAll project as a component of Simulation, Visualisation and Decision Support (SVD) Toolkit. In general the main goal of CoolEmAll is to provide advanced simulation, visualisation and decision support tools along with blueprints of computing building blocks for modular data centre environments. Once developed, these tools and blueprints should help to minimise the energy consumption, and consequently the CO2 emissions of the whole IT infrastructure with related facilities. The SVD Toolkit is designed to support the analysis and optimization of IT modern infrastructures. For the recent years the special attention has been paid for energy utilized by the data centers which considerable contributes to the data center operational costs.  Actual power usage and effectiveness of energy saving methods heavily depends on available resources, types of applications and workload properties. Therefore, intelligent resource management policies are gaining popularity when considering the energy efficiency of IT infrastructures.
+%Hence, SVD Toolkit integrates also workload management and scheduling policies to support complex modeling and optimization of modern data centres.
+
+%The main aim of DCWoRMS within CoolEmAll project is to enable studies of dynamic states of IT infrastructures, like power consumption and air throughput distribution, on the basis of changing workloads, resource model and energy-aware resource management policies.
+%In this context, DCWoRMS takes into account the specific workload and application characteristics as well as detailed resource parameters. It will benefit from the CoolEmAll benchmarks and classification of applications and workloads. In particular various types of workload, including data centre workloads using virtualization and HPC applications, may be considered. The knowledge concerning their performance and properties as well as information about their energy consumption and heat production will be used in simulations to study their impact on thermal issues and energy efficiency. Detailed resource characteristics, will be also provided according to the CoolEmAll blueprints. Based on this data, workload simulation will support evaluation process of various resource management approaches. These policies may include a wide spectrum of energy-aware strategies such as workload consolidation/migration, dynamic switching off nodes, DVFS and thermal-aware methods. In addition to typical approaches minimizing energy consumption, policies that prevent too high temperatures in the presence of limited cooling (or no cooling) may also be analyzed. Moreover, apart from the set of predefined strategies, new approaches can easily be applied and examined.
+%The outcome of the workload and resource management simulation phase is a distribution of power usage and air throughput for the computing models specified within the SVD Toolkit. These statistics may be analyzed directly by data centre designers and administrators and/or provided as an input to the CFD simulation phase. The former case allows studying how the above metrics change over time, while the latter harness CFD simulations to identify temperature differences between the computing modules, called hot spots. The goal of this scenario is to visualise the behavior of the temperature distribution within a server room with a number of racks for different types of executed workloads and for various policies used to manage these workloads.
 
 
 \section{Conclusions and future work}
 
-TODO - Conclusions and future research
+In this paper we presented a Data Center Workload and Resource Management Simulator (DCWoRMS) which enables modeling and simulation of computing infrastructures to estimate their performance, energy consumption, and energy-efficiency metrics for diverse workloads and management policies. DCWoRMS provides broad options of customization and combines detailed applications and workloads modeling with simulation of data center resources including their power usage and thermal behavior.
+We shown its energy-efficiency related features and proposed three methods of power usage modeling: static (fully defined by resource state), dynamic (defined by a function of parameters such as CPU frequency and load), and mapping (based on power usage of specific applications). We compared results of simulations to measurements of real servers and shown differences in accuracy and usability of these models. 
+We also demonstrated DCWoRMS capabilities to implement various resource management policies including workload scheduling and node power management.  The experimental studies we conducted shown that their  impact on overall energy-efficiency depends on a type of servers, their power usage in idle time, possibility of switching off nodes as well as level of load.
+DCWoRMS is a part of the Simulation, Visualisation and Decision Support (SVD) Toolkit being developed within the CoolEmAll project. The aim of this toolkit is, based on data center building blocks defined by the project, to analyze energy-efficiency of data centers taking into account various aspects such as heterogenous hardware architectures, applications, management policies, and cooling. DCWoRMS will take as an input open models data center building blocks and application profiles. DCWoRMS will be applied to evaluation of resource management approaches. These policies may include a wide spectrum of energy-aware strategies such as workload consolidation/migration, dynamic switching off nodes, DVFS and thermal-aware methods. Output of simulations will include distribution of servers' power usage in time along with estimations of server outlets air flow. These data will be passed to Computational Fluid Dynamics (CFD) simulations using OpenFOAM solver and to advanced 3D visualization. In this way users will be able to study energy-efficiency of a data center from a detailed analysis of workloads and policies to the impact on heat transfer and overall energy consumption. 
+Thus, future work on DCWoRMS will focus on more precise power, air-throughput, and thermal models. Additional research directions will include modeling application execution phases, adding predefined common HPC and cloud management policies and application performance and resource power models.
 
 \section*{Acknowledgement}
@@ -753 +820 @@
 \bibitem{fit4green} A. Berl, E. Gelenbe, M. di Girolamo, G. Giuliani, H. de Meer, M.-Q. Dang, K. Pentikousis. Energy-Efficient Cloud Computing. The Computer Journal, 53(7), 2010.
 
+\bibitem{e2dc12} Micha vor dem Berge, Georges Da Costa, Andreas Kopecki, Ariel Oleksiak, Jean-Marc Pierson, Tomasz Piontek, Eugen Volk, and Stefan Wesner. "Modeling and Simulation of Data Center Energy-Efficiency in CoolEmAll". Energy Efficient Data Centers, Lecture Notes in Computer Science Volume 7396, 2012, pp 25-36
+
 \bibitem{CloudSim} Rodrigo N. Calheiros, Rajiv Ranjan, Anton Beloglazov, Cesar A. F. De Rose, and Rajkumar Buyya, CloudSim: A Toolkit for Modeling and Simulation of Cloud Computing Environments and Evaluation of Resource Provisioning Algorithms, Software: Practice and Experience (SPE), Volume 41, Number 1, Pages: 23-50, ISSN: 0038-0644, Wiley Press, New York, USA, January, 2011.
 
@@ -767 +836 @@
 \bibitem{GreenCloud} D. Kliazovich, P. Bouvry, and S. U. Khan, A Packet-level Simulator of Energy- aware Cloud Computing Data Centers, Journal of Supercomputing, vol. 62, no. 3, pp. 1263-1283, 2012
 
-\bibitem{sla} S. Klingert, T. Schulze, C. Bunse. GreenSLAs for the Energy-efficient Management of Data Centres. 2nd International Conference on Energy-Efficient Computing and Networking (e-Energy), 2011.
+%\bibitem{sla} S. Klingert, T. Schulze, C. Bunse. GreenSLAs for the Energy-efficient Management of Data Centres. 2nd International Conference on Energy-Efficient Computing and Networking (e-Energy), 2011.
 
 \bibitem{GSSIM} S. Bak, M. Krystek, K. Kurowski, A. Oleksiak, W. Piatek and J. Weglarz, GSSIM - a Tool for Distributed Computing Experiments, Scientific Programming Journal, vol. 19, no. 4, pp. 231-251, 2011.
@@ -773 +842 @@
 \bibitem{GSSIM_Energy} M. Krystek, K. Kurowski, A. Oleksiak, W. Piatek, Energy-aware simulations with GSSIM. Proceedings of the COST Action IC0804 on Energy Efficiency in Large Scale Distributed Systems, 2010, pp. 55-58.
 
-\bibitem{hintemann} Hintemann, R., Fichter, K. (2010). Materialbestand der Rechenzentren in Deutschland, Eine Bestandsaufnahme zur Ermittlung von Ressourcen- und Energieeinsatz, UBA, Texte, 55/2010
+%\bibitem{hintemann} Hintemann, R., Fichter, K. (2010). Materialbestand der Rechenzentren in Deutschland, Eine Bestandsaufnahme zur Ermittlung von Ressourcen- und Energieeinsatz, UBA, Texte, 55/2010
 
 \bibitem{koomey}
 Koomey, Jonathan. 2008. "Worldwide electricity used in data centers." Environmental Research Letters. vol. 3, no. 034008. September 23
 
-
+\bibitem{fit4green_scheduler} Olli MÀmmelÀ, Mikko Majanen, Robert Basmadjian, Hermann De Meer, André Giesler, Willi Homberg, Energy-aware job scheduler for high-performance computing, Computer Science - Research and Development
+November 2012, Volume 27, Issue 4, pp 265-275
 
 % web links
@@ -797 +867 @@
 \bibitem{ecocooling} EcoCooling, http://www.ecocooling.org
 
-\bibitem{montblanc} The MontBlanc project website,  http://www.montblanc-project.eu/
+\bibitem{ff} Future Facilities, http://www.futurefacilities.com/
+
+%\bibitem{montblanc} The MontBlanc project website,  http://www.montblanc-project.eu/
 
 \bibitem{pue} The Green Grid Data Center Power Efficiency Metrics: PUE and DCiE, http://www.thegreengrid.org/Global/Content/white-papers/The-Green-Grid-Data-Center-Power-Efficiency-Metrics-PUE-and-DCiE