Changeset 1144 for DCWoRMS/branches/coolemall/src/simulator

Timestamp:

07/19/13 10:20:00 (12 years ago)

Author:

wojtekp

Message:

 

Location:

DCWoRMS/branches/coolemall/src/simulator/workload

Files:

• WorkloadLoader.java (modified) (8 diffs)
• reader/archive/AbstractWAParser.java (modified) (3 diffs)
• reader/archive/QcgWAJobReader.java (modified) (2 diffs)
• reader/archive/WAParser.java (modified) (2 diffs)
• reader/archive/swf/QcgSWFJobReader.java (modified) (4 diffs)

DCWoRMS/branches/coolemall/src/simulator/workload/WorkloadLoader.java

@@ -1 +1 @@
 package simulator.workload;
 
-import org.qcg.broker.schemas.jobdesc.Task;
-
+import java.io.BufferedReader;
+import java.io.FileReader;
 import java.io.IOException;
 import java.text.DateFormat;
@@ -58 +58 @@
         protected XsltTransformations xsltTransformation = null;
         
-        protected Map<String, JobDescription> jobGridletsMap;
-        protected Map<String, JobDescription> jobProfilesMap;
+        protected Map<String, JobDescription> jobsMap;
+        protected Map<String, JobDescription> applicationProfilesMap;
 
         public WorkloadLoader(XMLJobReader<org.qcg.broker.schemas.jobdesc.Job> xmlReader, WAReader<org.qcg.broker.schemas.jobdesc.Job> swfReader){
@@ -69 +69 @@
                 this.generatedJobsCnt = 0;
                 this.generatedTasksCnt = 0;
-                this.jobGridletsMap = new TreeMap<String, JobDescription>();
-                this.jobProfilesMap = new TreeMap<String, JobDescription>();
+                this.jobsMap = new TreeMap<String, JobDescription>();
+                this.applicationProfilesMap = new TreeMap<String, JobDescription>();
                 try {
                         this.xsltTransformation = new XsltTransformations();
@@ -97 +97 @@
                 }
                 
-                // needed for Gridlet class
+                // needed for Executable class
                 
                 String jobId = jobDescription.getJobId();
@@ -107 +107 @@
                         String waTaskDesc[] = localWAParser.readTask(jobId, taskDesc.getTaskId());
                         
-                        try{
-                                Integer.parseInt(taskDesc.getTaskId());
-                        } catch (NumberFormatException e){
-                                return jobDescription;
-                        }
                         if(waTaskDesc != null) {
                                 
@@ -188 +183 @@
                         while((jobDesc = this.xmlJobReader.readRaw()) != null){
                                 jobDesc = this.xsltTransformation.extendJobDescription(jobDesc);
-                                int prevJobCnt = generatedJobsCnt;
-                                job = createJobDescription(jobDesc, puSpeed);
-                                if(job != null && prevJobCnt != generatedJobsCnt)
-                                        this.jobGridletsMap.put(job.getJobId(), job);
-                                else if (prevJobCnt == generatedJobsCnt){
-                                        Task taskPattern = job.getDescription().getTask(0);
-                                        this.jobProfilesMap.put(taskPattern.getId(), job);
-                                }
-                        }
-                } else {                                                // use swf job reader. Job created by this reader does not require xslt transformation
-                        while((jobDesc = this.waReader.readRaw()) != null){
                                 job = createJobDescription(jobDesc, puSpeed);
                                 if(job != null)
-                                        this.jobGridletsMap.put(job.getJobId(), job);
-                        }
-                }
-                
-                if(generatedJobsCnt == 0){
+                                        this.jobsMap.put(job.getJobId(), job);
+                        }
+                } else {                                                // use swf job reader. Job created by this reader does not require xslt transformation
+                        
+                        for(String key: waReader.getParser().getAppMapping().keySet()){
+                                jobDesc = loadApplicationProfile(waReader.getParser().getAppMapping().get(key));
+                                jobDesc = this.xsltTransformation.extendJobDescription(jobDesc);
+                                        
+                                try {
+                                        job = this.xsltTransformation.splitJobToTasks(jobDesc);
+                                } catch (Exception e) {
+                                        throw new IOException(e.getMessage());
+                                }
+                                
+                                this.applicationProfilesMap.put(key, job);
+                        }
+                        
                         while((jobDesc = this.waReader.readRaw()) != null){
                                 QcgWAJobReader qcgReader = (QcgWAJobReader)waReader;
-                                jobDesc = qcgReader.mergeSwfAndXmlProfile(jobProfilesMap, jobDesc);
+                                jobDesc = qcgReader.mergeSwfAndXmlProfile(applicationProfilesMap, jobDesc);
                                 job = createJobDescription(jobDesc, puSpeed);
                                 if(job != null)
-                                        this.jobGridletsMap.put(job.getJobId(), job);
+                                        this.jobsMap.put(job.getJobId(), job);
                         }
                 }
@@ -268 +264 @@
         public List<JobDescription> getJobs(){
                 ArrayList<JobDescription> list = 
-                        new ArrayList<JobDescription>(this.jobGridletsMap.values());
+                        new ArrayList<JobDescription>(this.jobsMap.values());
                 return list;
         }
@@ -276 +272 @@
         }
 
-        protected JobDescription createJobDescription(String jobDesc, long puSpeed, Map<String, JobDescription> jobProfilesMap) throws IOException{
-                
-                JobDescription jobDescription = null;
-                try {
-                        jobDescription = this.xsltTransformation.splitJobToTasks(jobDesc);
-                } catch (Exception e) {
-                        throw new IOException(e.getMessage());
-                }
-                
-                // look for any 
-                
-                if(jobDescription.size() == 0){
-                        if(log.isWarnEnabled())
-                                log.warn("Omitting tasks creation for job "+jobDescription.getDescription().getId()+". This job contains no tasks.");
-                        return null;
-                }
-
-                String jobId = jobDescription.getJobId();
-                int parserType = localWAParser.getType();
-                
-                for(int i = 0; i < jobDescription.size(); i++){
-                        TaskDescription taskDesc = jobDescription.get(i);
-                        
-                        String waTaskDesc[] = localWAParser.readTask(jobId, taskDesc.getTaskId());
-                        
-                        if(waTaskDesc != null) {
-                                
-                                if(parserType == 0){
-                                
-                                        taskDesc.setUserDn(waTaskDesc[SWFFields.DATA_USER_ID]);
-                                        
-                                        long timeValue = Long.parseLong(waTaskDesc[SWFFields.DATA_SUBMIT_TIME]);
-                                        taskDesc.setSubmissionTime(timeValue);
-                                        
-                                        long waitTime = Long.parseLong(waTaskDesc[SWFFields.DATA_WAIT_TIME]);
-                                        taskDesc.setWorkloadLogWaitTime(waitTime);
-                                        
-                                        timeValue = Long.parseLong(waTaskDesc[SWFFields.DATA_RUN_TIME]);
-                                        if(timeValue <= 0)
-                                                return null;
-                                        
-                                        long allocProcesors = Long.parseLong(waTaskDesc[SWFFields.DATA_NUMBER_OF_ALLOCATED_PROCESSORS]);
-                                        if(allocProcesors <= 0)
-                                                return null;
-                                        
-                                        timeValue = timeValue * puSpeed * allocProcesors;
-                                        taskDesc.setTaskLength(timeValue);
-
-                                } else if(parserType == 1){
-                                
-                                        taskDesc.setUserDn(waTaskDesc[GWFFields.DATA_USER_ID]);
-                                        
-                                        long timeValue = Long.parseLong(waTaskDesc[GWFFields.DATA_SUBMIT_TIME]);
-                                        taskDesc.setSubmissionTime(timeValue);
-                                        
-                                        long waitTime = Long.parseLong(waTaskDesc[GWFFields.DATA_WAIT_TIME]);
-                                        taskDesc.setWorkloadLogWaitTime(waitTime);
-                                        
-                                        timeValue = Long.parseLong(waTaskDesc[GWFFields.DATA_RUN_TIME]);
-                                        if(timeValue <= 0)
-                                                return null;
-                                        
-                                        long allocProcesors = Long.parseLong(waTaskDesc[GWFFields.DATA_NPROCS]);
-                                        if(allocProcesors <= 0)
-                                                return null;
-                                        
-                                        timeValue = timeValue * puSpeed * allocProcesors;
-                                        taskDesc.setTaskLength(timeValue);
-                                }
-                        }
-                        /*JobDescription xmlJobDescription = jobProfilesMap.get(taskDesc.getDescription().getExecution().getExecutable().getApplication().getName());
-                        if(xmlJobDescription != null){
-                                Task patternTask = xmlJobDescription.getDescription().getTask(0);
-                                if(patternTask != null){
-                                        ResourceConsumptionProfileType rcp = patternTask.getExecution().getResourceConsumptionProfile();
-                                        taskDesc.getDescription().getExecution().setResourceConsumptionProfile(rcp);
-                                }
-                        }*/
-                        this.generatedTasksCnt++;
-                        
-                }
-                this.generatedJobsCnt++;
-                
-                return jobDescription;
-        }
-        
+        public String loadApplicationProfile(String fileName) throws IOException{
+                
+                BufferedReader reader = null;
+                StringBuffer buffer = new StringBuffer();
+
+                try {
+                        reader = new BufferedReader(new FileReader(fileName));
+                        String line = null;
+                        
+                        while((line = reader.readLine()) != null){
+                                buffer.append(line);
+                        }
+                        
+                } finally{
+                        reader.close();
+                }
+                
+                return buffer.toString();
+        }
 }

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/AbstractWAParser.java

@@ -4 +4 @@
 import java.io.RandomAccessFile;
 import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -150 +151 @@
                                 String [] valueData = value.split(" ");
                                 String appId = valueData[0];
-                                String appName = valueData[1];
-                                appMapping.put(appId, appName);
+                                String pathToAppProfile = valueData[1];
+                                appMapping.put(appId, pathToAppProfile);
                                 continue;
                         }
@@ -295 +296 @@
         
         
-        public String getAppMapping(String appId) {
-                String appName = appMapping.get(appId);
-                return appName;
+        public Map<String, String> getAppMapping() {
+                return appMapping;
         }
         

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/QcgWAJobReader.java

@@ -109 +109 @@
         }
 
-        public String mergeSwfAndXmlProfile( Map<String, JobDescription> jobProfilesMap, String swfJobDesc) throws IOException{
+        public String mergeSwfAndXmlProfile(Map<String, JobDescription> jobProfilesMap, String swfJobDesc) throws IOException{
                 
                 StringReader reader = new StringReader(swfJobDesc);
@@ -123 +123 @@
                 }
                 
-                JobDescription xmlJobDescription = jobProfilesMap.get(job.getTask(0).getExecution().getExecutable().getApplication().getName());
-                if(xmlJobDescription != null){
-                        Task patternTask = xmlJobDescription.getDescription().getTask(0);
-                        if(patternTask != null){
-                                ResourceConsumptionProfileType rcp = patternTask.getExecution().getResourceConsumptionProfile();
-                                job.getTask(0).getExecution().setResourceConsumptionProfile(rcp);
+                for(int i = 0; i < job.getTaskCount(); i++){
+                        
+                        JobDescription xmlJobDescription = jobProfilesMap.get(job.getTask(i).getExecution().getExecutable().getApplication().getAppProperty(0).getContent());
+                        if(xmlJobDescription != null){
+                                for(int j = 0; j < xmlJobDescription.size(); j++){
+                                        Task patternTask = xmlJobDescription.getDescription().getTask(j);
+                                        if(patternTask != null){
+                                                ResourceConsumptionProfileType rcp = patternTask.getExecution().getResourceConsumptionProfile();
+                                                job.getTask(j).getExecution().setResourceConsumptionProfile(rcp);
+                                        }
+                                }
                         }
                 }

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/WAParser.java

@@ -3 +3 @@
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Map;
 
 import simulator.workload.exceptons.NoSuchCommentException;
@@ -44 +45 @@
         public int getType();
         
-        public String getAppMapping(String appId);
+        public Map<String, String> getAppMapping();
 }

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/swf/QcgSWFJobReader.java

@@ -1 +1 @@
 package simulator.workload.reader.archive.swf;
 
+import org.qcg.broker.schemas.jobdesc.AppProperty;
 import org.qcg.broker.schemas.jobdesc.Application;
 import org.qcg.broker.schemas.jobdesc.ComputingResourceBaseTypeChoice;
@@ -40 +41 @@
         private Log log = LogFactory.getLog(QcgSWFJobReader.class);
         
+        private static String DEFAULT_PATH_TO_APP_PROFILES = "";
+        
         protected String currntJobID;
         
@@ -218 +221 @@
                                         
                                 case SWFFields.DATA_EXECUTABLE_NUMBER:
-                                        String appName = waParser.getAppMapping(String.valueOf(value));
-                                        if(appName == null){
+                                        String pathToAppProfile = waParser.getAppMapping().get(String.valueOf(value));
+                                        String appName = null;
+                                        if(pathToAppProfile == null){
                                                 if(value != -1){
                                                         appName = String.valueOf(value);
@@ -225 +229 @@
                                                         appName = appTypeGen.randomAppType();
                                                 }
+                                                pathToAppProfile =  DEFAULT_PATH_TO_APP_PROFILES + appName;
+                                        } else {
+                                                String fullAppName = pathToAppProfile.substring(pathToAppProfile.lastIndexOf('/') + 1, pathToAppProfile.length());
+                                                if(fullAppName != null)
+                                                        appName = fullAppName.substring(0, fullAppName.lastIndexOf('.'));
+                                                else appName = fullAppName;
                                         }
                                         ExecutionType execType = new ExecutionType();
                                         Executable executable = new Executable();
                                         Application application = new Application();
+                                        AppProperty appProp = new AppProperty();
+                                        appProp.setName("appId");
+                                        appProp.setContent(String.valueOf(value));
+                                        application.addAppProperty(appProp);
+                                        /*appProp = new AppProperty();
+                                        appProp.setName("appProfileLocation");
+                                        appProp.setContent(pathToAppProfile);
+                                        application.addAppProperty(appProp);*/
                                         application.setName(appName);
                                         executable.setApplication(application);