Changeset 1207 for DCWoRMS/branches/coolemall/src/simulator/workload

Timestamp:

11/26/13 11:56:07 (11 years ago)

Author:

wojtekp

Message:

 

Location:

DCWoRMS/branches/coolemall/src/simulator/workload

Files:

• WorkloadLoader.java (modified) (6 diffs)
• reader/archive/AbstractWAParser.java (modified) (4 diffs)
• reader/archive/QcgWAJobReader.java (modified) (3 diffs)
• reader/archive/WAParser.java (modified) (1 diff)
• reader/archive/swf/QcgSWFJobReader.java (modified) (1 diff)

DCWoRMS/branches/coolemall/src/simulator/workload/WorkloadLoader.java

@@ -17 +17 @@
 import javax.xml.xpath.XPathExpressionException;
 
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -32 +33 @@
 import simulator.workload.reader.archive.swf.SWFFields;
 import simulator.workload.reader.xmlJob.XMLJobReader;
+import dcworms.schedframe.scheduling.utils.ApplicationProfileDescription;
 import dcworms.schedframe.scheduling.utils.JobDescription;
 import dcworms.schedframe.scheduling.utils.TaskDescription;
@@ -56 +58 @@
         
         protected Map<String, JobDescription> jobsMap;
-        protected Map<String, JobDescription> applicationProfilesMap;
-
-        public WorkloadLoader(XMLJobReader<org.qcg.broker.schemas.jobdesc.Job> xmlReader, WAReader<org.qcg.broker.schemas.jobdesc.Job> swfReader){
+        
+        protected String appProfilesFolder;
+        protected Map<String, ApplicationProfileDescription> applicationProfiles;
+
+        public WorkloadLoader(XMLJobReader<org.qcg.broker.schemas.jobdesc.Job> xmlReader, WAReader<org.qcg.broker.schemas.jobdesc.Job> swfReader, String appProfilesFolder){
                 if(swfReader == null){
                         throw new RuntimeException("Swf reader is required to build proper tasks.");
@@ -67 +71 @@
                 this.generatedTasksCnt = 0;
                 this.jobsMap = new TreeMap<String, JobDescription>();
-                this.applicationProfilesMap = new TreeMap<String, JobDescription>();
+                this.appProfilesFolder = appProfilesFolder;
+                this.applicationProfiles = new TreeMap<String, ApplicationProfileDescription>();
                 try {
                         this.xsltTransformation = new XsltTransformations();
@@ -187 +192 @@
 
                 // determine which reader should be used
-                String jobDesc = null;
-                JobDescription job = null;
-                        
+                String xmlJob = null;
+                JobDescription jobDesc = null;
+                        
+                for(String key: waReader.getParser().getAppProfilesLocation().keySet()){
+                        String xmlApp = loadApplicationProfile(waReader.getParser().getAppProfilesLocation().get(key));
+                        xmlApp = this.xsltTransformation.extendJobDescription(xmlApp);
+                        ApplicationProfileDescription app = this.xsltTransformation.getApplicationProfileDescription(xmlApp);
+                        this.applicationProfiles.put(key, app);
+                }
+                
                 if(this.xmlJobReader != null){ // use xml job reader. Xml job require xslt transformation
-                        while((jobDesc = this.xmlJobReader.readRaw()) != null){
-                                jobDesc = this.xsltTransformation.extendJobDescription(jobDesc);
-                                job = createJobDescription(jobDesc, puSpeed);
-                                if(job != null)
-                                        this.jobsMap.put(job.getJobId(), job);
+                        while((xmlJob = this.xmlJobReader.readRaw()) != null){
+                                xmlJob = this.xsltTransformation.extendJobDescription(xmlJob);
+                                jobDesc = createJobDescription(xmlJob, puSpeed);
+                                if(jobDesc != null)
+                                        this.jobsMap.put(jobDesc.getJobId(), jobDesc);
                         }
                 } else {                                                // use swf job reader. Job created by this reader does not require xslt transformation
                         
-                        for(String key: waReader.getParser().getAppMapping().keySet()){
-                                jobDesc = loadApplicationProfile(waReader.getParser().getAppMapping().get(key));
-                                jobDesc = this.xsltTransformation.extendJobDescription(jobDesc);
-                                        
-                                try {
-                                        job = this.xsltTransformation.splitJobToTasks(jobDesc);
-                                } catch (Exception e) {
-                                        throw new IOException(e.getMessage());
-                                }
-                                
-                                this.applicationProfilesMap.put(key, job);
-                        }
-                        
-                        while((jobDesc = this.waReader.readRaw()) != null){
+                        while((xmlJob = this.waReader.readRaw()) != null){
                                 QcgWAJobReader qcgReader = (QcgWAJobReader)waReader;
-                                if(!applicationProfilesMap.isEmpty())
-                                        jobDesc = qcgReader.mergeSwfAndXmlProfile(applicationProfilesMap, jobDesc);
-                                job = createJobDescription(jobDesc, puSpeed);
-                                /*if(job!= null && !applicationProfilesMap.isEmpty()){
-                                        for(int i = 0; i < job.getDescription().getTaskCount(); i++){
-                                                try {
-                                                        JobDescription xmlJobDescription = applicationProfilesMap.get(job.getDescription().getTask(i).getExecution().getExecutable().getApplication().getAppProperty(0).getContent());
-                                                        if(xmlJobDescription != null){
-                                                                for(int j = 0; j < xmlJobDescription.size(); j++){
-                                                                        Task patternTask = xmlJobDescription.getDescription().getTask(j);
-                                                                        if(patternTask != null){
-                                                                                ResourceConsumptionProfileType rcp = patternTask.getExecution().getResourceConsumptionProfile();
-                                                                                job.getDescription().getTask(j).getExecution().setResourceConsumptionProfile(rcp);
-                                                                        }
-                                                                }
-                                                        }       
-                                                } catch (Exception e){
-                                                        continue;
-                                                }
-                                        }                                       
-                                }
-                                */
-                                if(job != null)
-                                        this.jobsMap.put(job.getJobId(), job);
+                                if(!applicationProfiles.isEmpty())
+                                        xmlJob = qcgReader.mergeSwfAndAppProfile(applicationProfiles, xmlJob);
+                                jobDesc = createJobDescription(xmlJob, puSpeed);
+                                if(jobDesc != null)
+                                        this.jobsMap.put(jobDesc.getJobId(), jobDesc);
                         }
                 }
@@ -302 +282 @@
         }
 
-        public String loadApplicationProfile(String fileName) throws IOException{
+        public String loadApplicationProfile(String pathToAppProfile) throws IOException{
                 
                 BufferedReader reader = null;
                 StringBuffer buffer = new StringBuffer();
 
-                try {
-                        reader = new BufferedReader(new FileReader(fileName));
+                String localPathToAppProfile;
+                
+                if(appProfilesFolder != null) {
+                        String folderName = FilenameUtils.getFullPath(appProfilesFolder);
+                        String fileName = FilenameUtils.getName(pathToAppProfile);
+                        localPathToAppProfile = folderName + fileName;
+                } else {
+                        localPathToAppProfile = pathToAppProfile;
+                }
+                try {
+                        reader = new BufferedReader(new FileReader(localPathToAppProfile ));
                         String line = null;
                         

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/AbstractWAParser.java

@@ -33 +33 @@
         protected HashMap<String, String> reverseIdMapping; // key - xmlJobId_xmlTaskId, value - swf job id
         protected HashMap<String, Long> jobIndex;
-        protected HashMap<String, String> appMapping;
+        protected HashMap<String, String> appProfilesLocation;
         protected String fields[];
         protected int fieldsNo;
@@ -46 +46 @@
                 this.reverseIdMapping = new HashMap<String, String>();
                 this.jobIndex = new HashMap<String, Long>();
-                this.appMapping = new HashMap<String, String>();
+                this.appProfilesLocation = new HashMap<String, String>();
                 this.headerLoaded = false;
                 this.buildIndex = true;
                 this.fieldsNo = 18;
+                
         }
 
@@ -152 +153 @@
                                 String appId = valueData[0];
                                 String pathToAppProfile = valueData[1];
-                                appMapping.put(appId, pathToAppProfile);
+                                appProfilesLocation.put(appId, pathToAppProfile);
                                 continue;
                         }
@@ -296 +297 @@
         
         
-        public Map<String, String> getAppMapping() {
-                return appMapping;
+        public Map<String, String> getAppProfilesLocation() {
+                return appProfilesLocation;
         }
         

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/QcgWAJobReader.java

@@ -15 +15 @@
 
 import simulator.workload.reader.archive.swf.SWFFields;
+
+import org.qcg.broker.schemas.jobdesc.FileType;
 import org.qcg.broker.schemas.jobdesc.ResourceConsumptionProfileType;
 import org.qcg.broker.schemas.jobdesc.Task;
 import org.qcg.broker.schemas.jobdesc.Workflow;
 
-import dcworms.schedframe.scheduling.utils.JobDescription;
+import dcworms.schedframe.scheduling.utils.ApplicationProfileDescription;
 
 /**
@@ -116 +118 @@
         }
 
-        public String mergeSwfAndXmlProfile(Map<String, JobDescription> applicationProfilesMap, String swfJobDesc) throws IOException{
+        public String mergeSwfAndAppProfile(Map<String, ApplicationProfileDescription> applicationProfiles, String swfJobDesc) throws IOException{
                 
                 StringReader reader = new StringReader(swfJobDesc);
@@ -132 +134 @@
                 for(int i = 0; i < job.getTaskCount(); i++){
                         try {
-                                JobDescription xmlJobDescription = applicationProfilesMap.get(job.getTask(i).getExecution().getExecutable().getApplication().getAppProperty(0).getContent());
-                                if(xmlJobDescription != null){
-                                        for(int j = 0; j < xmlJobDescription.size(); j++){
-                                                Task patternTask = xmlJobDescription.getDescription().getTask(j);
-                                                if(patternTask != null){
-                                                        ResourceConsumptionProfileType rcp = patternTask.getExecution().getResourceConsumptionProfile();
-                                                        job.getTask(j).getExecution().setResourceConsumptionProfile(rcp);
-                                                }
+                                ApplicationProfileDescription appProfileDescription = applicationProfiles.get(job.getTask(i).getExecution().getExecutable().getApplication().getAppProperty(0).getContent());
+                                if(appProfileDescription != null){
+                                        Task patternTask = appProfileDescription.getDescription();
+                                        if(patternTask != null){
+                                                ResourceConsumptionProfileType[] rcp = patternTask.getExecution().getResourceConsumptionProfile();
+                                                job.getTask(i).getExecution().setResourceConsumptionProfile(rcp);
+                                                FileType script = new FileType();
+                                                script.setName(patternTask.getId());
+                                                job.getTask(i).getExecution().addStdin(script);
                                         }
                                 }       

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/WAParser.java

@@ -45 +45 @@
         public int getType();
         
-        public Map<String, String> getAppMapping();
+        public Map<String, String> getAppProfilesLocation();
 }

DCWoRMS/branches/coolemall/src/simulator/workload/reader/archive/swf/QcgSWFJobReader.java

@@ -221 +221 @@
                                         
                                 case SWFFields.DATA_EXECUTABLE_NUMBER:
-                                        String pathToAppProfile = waParser.getAppMapping().get(String.valueOf(value));
+                                        String pathToAppProfile = waParser.getAppProfilesLocation().get(String.valueOf(value));
                                         String appName = null;
                                         if(pathToAppProfile == null){