Changeset 883 for DCWoRMS/branches/coolemall/src/simulator/workload/WorkloadLoader.java

Timestamp:

02/26/13 08:41:50 (12 years ago)

Author:

wojtekp

Message:

 

Location:

DCWoRMS/branches/coolemall

Files:

• . (copied) (copied from DCWoRMS/trunk)
• src/simulator/workload/WorkloadLoader.java (modified) (10 diffs)

DCWoRMS/branches/coolemall/src/simulator/workload/WorkloadLoader.java

@@ -2 +2 @@
 
 import org.qcg.broker.schemas.jobdesc.QcgJob;
+import org.qcg.broker.schemas.jobdesc.ResourceConsumptionProfileType;
+import org.qcg.broker.schemas.jobdesc.Task;
 
 import java.io.IOException;
@@ -27 +29 @@
 
 
+import schedframe.scheduling.tasks.Job;
 import simulator.utils.XsltTransformations;
 import simulator.workload.exceptons.NoSuchCommentException;
 import simulator.workload.reader.archive.AbstractWAParser;
+import simulator.workload.reader.archive.QcgWAJobReader;
 import simulator.workload.reader.archive.WAFields;
 import simulator.workload.reader.archive.WAReader;
@@ -58 +62 @@
         
         protected Map<String, JobDescription> jobGridletsMap;
-        
+        protected Map<String, JobDescription> jobProfilesMap;
+
         public WorkloadLoader(XMLJobReader<QcgJob> xmlReader, WAReader<QcgJob> swfReader){
                 if(swfReader == null){
-                        throw new RuntimeException("Swf reader is required to build proper gridlerts.");
+                        throw new RuntimeException("Swf reader is required to build proper tasks.");
                 }
                 this.xmlJobReader = xmlReader;
@@ -68 +73 @@
                 this.generatedTasksCnt = 0;
                 this.jobGridletsMap = new TreeMap<String, JobDescription>();
+                this.jobProfilesMap = new TreeMap<String, JobDescription>();
                 try {
                         this.xsltTransformation = new XsltTransformations();
@@ -90 +96 @@
                 if(jobDescription.size() == 0){
                         if(log.isWarnEnabled())
-                                log.warn("Omiting job gridlet creation for job "+jobDescription.getDescription().getAppId()+". This job contains no tasks.");
+                                log.warn("Omitting tasks creation for job "+jobDescription.getDescription().getAppId()+". This job contains no tasks.");
                         return null;
                 }
@@ -103 +109 @@
                         
                         String waTaskDesc[] = localWAParser.readTask(jobId, taskDesc.getTaskId());
+                        
+                        try{
+                                Integer.parseInt(taskDesc.getTaskId());
+                        } catch (NumberFormatException e){
+                                return jobDescription;
+                        }
                         if(waTaskDesc != null) {
                                 
@@ -125 +137 @@
                                         timeValue = timeValue * puSpeed * allocProcesors;
                                         taskDesc.setTaskLength(timeValue);
+                                        
+                                        //if(taskDesc.getDescription().getExecution()!= n
 
                                 } else if(parserType == 1){
@@ -177 +191 @@
                         while((jobDesc = this.xmlJobReader.readRaw()) != null){
                                 jobDesc = this.xsltTransformation.extendJobDescription(jobDesc);
+                                int prevJobCnt = generatedJobsCnt;
                                 job = createJobDescription(jobDesc, puSpeed);
-                                if(job != null)
+                                if(job != null && prevJobCnt != generatedJobsCnt)
                                         this.jobGridletsMap.put(job.getJobId(), job);
+                                else if (prevJobCnt == generatedJobsCnt){
+                                        Task taskPattern = job.getDescription().getTask(0);
+                                        this.jobProfilesMap.put(taskPattern.getTaskId(), job);
+                                }
                         }
                 } else {                                                // use swf job reader. Job created by this reader does not require xslt transformation
@@ -188 +207 @@
                         }
                 }
+                
+                if(generatedJobsCnt == 0){
+                        while((jobDesc = this.waReader.readRaw()) != null){
+                                QcgWAJobReader qcgReader = (QcgWAJobReader)waReader;
+                                jobDesc = qcgReader.mergeSwfAndXmlProfile(jobProfilesMap, jobDesc);
+                                job = createJobDescription(jobDesc, puSpeed);
+                                if(job != null)
+                                        this.jobGridletsMap.put(job.getJobId(), job);
+                        }
+                }
 
                 this.localWAParser.close();
@@ -249 +278 @@
                 return simStartTimeDefined;
         }
+
+        protected JobDescription createJobDescription(String jobDesc, long puSpeed, Map<String, JobDescription> jobProfilesMap) throws IOException{
+                
+                JobDescription jobDescription = null;
+                try {
+                        jobDescription = this.xsltTransformation.splitJobToTasks(jobDesc);
+                } catch (Exception e) {
+                        throw new IOException(e.getMessage());
+                }
+                
+                // look for any 
+                
+                if(jobDescription.size() == 0){
+                        if(log.isWarnEnabled())
+                                log.warn("Omitting tasks creation for job "+jobDescription.getDescription().getAppId()+". This job contains no tasks.");
+                        return null;
+                }
+
+                String jobId = jobDescription.getJobId();
+                int parserType = localWAParser.getType();
+                
+                for(int i = 0; i < jobDescription.size(); i++){
+                        TaskDescription taskDesc = jobDescription.get(i);
+                        
+                        String waTaskDesc[] = localWAParser.readTask(jobId, taskDesc.getTaskId());
+                        
+                        if(waTaskDesc != null) {
+                                
+                                if(parserType == 0){
+                                
+                                        taskDesc.setUserDn(waTaskDesc[SWFFields.DATA_USER_ID]);
+                                        
+                                        long timeValue = Long.parseLong(waTaskDesc[SWFFields.DATA_SUBMIT_TIME]);
+                                        taskDesc.setSubmissionTime(timeValue);
+                                        
+                                        long waitTime = Long.parseLong(waTaskDesc[SWFFields.DATA_WAIT_TIME]);
+                                        taskDesc.setWorkloadLogWaitTime(waitTime);
+                                        
+                                        timeValue = Long.parseLong(waTaskDesc[SWFFields.DATA_RUN_TIME]);
+                                        if(timeValue <= 0)
+                                                return null;
+                                        
+                                        long allocProcesors = Long.parseLong(waTaskDesc[SWFFields.DATA_NUMBER_OF_ALLOCATED_PROCESSORS]);
+                                        if(allocProcesors <= 0)
+                                                return null;
+                                        
+                                        timeValue = timeValue * puSpeed * allocProcesors;
+                                        taskDesc.setTaskLength(timeValue);
+
+                                } else if(parserType == 1){
+                                
+                                        taskDesc.setUserDn(waTaskDesc[GWFFields.DATA_USER_ID]);
+                                        
+                                        long timeValue = Long.parseLong(waTaskDesc[GWFFields.DATA_SUBMIT_TIME]);
+                                        taskDesc.setSubmissionTime(timeValue);
+                                        
+                                        long waitTime = Long.parseLong(waTaskDesc[GWFFields.DATA_WAIT_TIME]);
+                                        taskDesc.setWorkloadLogWaitTime(waitTime);
+                                        
+                                        timeValue = Long.parseLong(waTaskDesc[GWFFields.DATA_RUN_TIME]);
+                                        if(timeValue <= 0)
+                                                return null;
+                                        
+                                        long allocProcesors = Long.parseLong(waTaskDesc[GWFFields.DATA_NPROCS]);
+                                        if(allocProcesors <= 0)
+                                                return null;
+                                        
+                                        timeValue = timeValue * puSpeed * allocProcesors;
+                                        taskDesc.setTaskLength(timeValue);
+                                }
+                        }
+                        /*JobDescription xmlJobDescription = jobProfilesMap.get(taskDesc.getDescription().getExecution().getExecutable().getApplication().getName());
+                        if(xmlJobDescription != null){
+                                Task patternTask = xmlJobDescription.getDescription().getTask(0);
+                                if(patternTask != null){
+                                        ResourceConsumptionProfileType rcp = patternTask.getExecution().getResourceConsumptionProfile();
+                                        taskDesc.getDescription().getExecution().setResourceConsumptionProfile(rcp);
+                                }
+                        }*/
+                        this.generatedTasksCnt++;
+                        
+                }
+                this.generatedJobsCnt++;
+                
+                return jobDescription;
+        }
         
 }