Changeset 26 for trunk/ll_drmaa/job.c

Timestamp:

11/11/12 15:06:16 (13 years ago)

Author:

mmamonski

Message:

SupMUC on site fixes: 1. Polling mode 2. Handling missing jobs 3. monitor -> drmaa_monitor 4. force stderr file creation

File:

• trunk/ll_drmaa/job.c (modified) (6 diffs)

trunk/ll_drmaa/job.c

@@ -35 +35 @@
 
 #include <llapi.h>
+
+#define LL_DRMAA_MAX_MISSING_TIME (60)
 
 static void
@@ -287 +289 @@
                 }
 
-                data = ll_get_objs(job, LL_SCHEDD, NULL, &obj_count, &err_code);
+                data = ll_get_objs(job, LL_CM, NULL, &obj_count, &err_code);
                 if (data == NULL) {
-                        fsd_log_debug(("Code: %d ll_get_objs() returns NULL. %s",lldrmaa_map_get_objs(err_code), lldrmaa_err_get_objs(err_code) ));
-                        /* This error means that there is no info in LL_SCHEDD but this job have probably ended and will be detected by wait_thread. This won't be fsd_log_warning because displays too often during program execution */
+                        if (err_code == LL_GET_OBJS_NO_OBJECTS_ERR)
+                          {
+                                if (self->state != DRMAA_PS_UNDETERMINED) 
+                                  {
+                                        fsd_log_info(("Job %s missing. Assuming finished", self->job_id));
+                                        self->state = DRMAA_PS_DONE;
+                                        self->exit_status = 0;
+                                  }
+                                else if (llself->missing_time == 0)
+                                 {
+                                        llself->missing_time = time(NULL);
+                                        fsd_log_debug(("Job %s missing for the first time", self->job_id)); /* Job may not yet be visible in LL */
+                                 }
+                                else if (time(NULL) - llself->missing_time > LL_DRMAA_MAX_MISSING_TIME)
+                                 {
+                                        fsd_log_error(("Job %s missing for more then %d seconds. Assuming failef", self->job_id, LL_DRMAA_MAX_MISSING_TIME)); 
+                                        /* Job may not yet be visible in LL */
+                                        self->state = DRMAA_PS_FAILED;
+                                        self->exit_status = -1;
+                                 }
+                                else
+                                 {
+                                        fsd_log_debug(("Job %s still missing", self->job_id));
+                                 }
+                          }
+                        else
+                          {
+                                fsd_log_error(("Code: %d,%d ll_get_objs() returns NULL. %s", err_code, lldrmaa_map_get_objs(err_code), lldrmaa_err_get_objs(err_code) ));
+                          }
                 } else {
 
@@ -312 +341 @@
                         }
 
-                        llself->read_job_info( self, step_state, hold_type);
-
-                        fsd_log_info(("LL State: %d -> DRMAA: %s",step_state,drmaa_job_ps_to_str(self->state)));
+                        llself->read_job_info(self, step_state, hold_type);
+
+                        fsd_log_info(("LL State: %d -> DRMAA: %s", step_state, drmaa_job_ps_to_str(self->state)));
                 }
         }
@@ -352 +381 @@
         self->super.control = lldrmaa_job_control;
         self->super.update_status = lldrmaa_job_update_status;
+        self->missing_time = 0;
         self->read_job_info_mon = lldrmaa_job_read_job_info_mon;
         self->read_job_info = lldrmaa_job_read_job_info;
@@ -471 +501 @@
                 }
 
-                TRY
-                {
-                        const char *command = NULL;
-                        char *command_expanded = NULL;
-                        const char *const *i;
-                        int j;
-
-                        /* remote command */
-                        command = jt->get_attr( jt, DRMAA_REMOTE_COMMAND );
-                        if( command == NULL )
-                                fsd_exc_raise_msg(
-                                                FSD_DRMAA_ERRNO_CONFLICTING_ATTRIBUTE_VALUES,
-                                                "drmaa_remote_command not set for job template"
-                                                );
-
-                        command_expanded = expand->expand( expand, fsd_strdup(command), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD );
-
-                        fprintf(fd,"# @ executable = %s\n", command_expanded);
-                        fsd_log_debug(("# @ executable = %s\n", command_expanded));
-
-                        fsd_free(command_expanded);
-
-                        /* arguments list */
-                        vector = jt->get_v_attr( jt, DRMAA_V_ARGV );
-
-                        if( vector )
-                        {
-                                fprintf(fd,"# @ arguments =");
-                                fsd_log_debug(("# @ arguments ="));
-
-                                for( i = vector, j = 2;  *i;  i++, j++ )
-                                {
-                                        char *arg_expanded = expand->expand( expand, fsd_strdup(*i), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD );
-
-                                        fprintf(fd," '%s'", arg_expanded);
-                                        fsd_log_debug(("%s", arg_expanded));
-
-                                        fsd_free(arg_expanded);
-                                }
-                        }
-
-                        fprintf(fd," \n");
-                }
-                END_TRY
-
                 /* start time */
                 value = jt->get_attr( jt, DRMAA_START_TIME );
@@ -779 +764 @@
                                 fprintf(fd,"# @ queue\n");
                                 fsd_log_debug(("# @ queue"));
+                                
+                                fprintf(fd,"\n");
+                                fprintf(fd,"echo >&2\n"); /* this line forces creation of stderr file */
+
+                                TRY
+                                {
+                                        const char *command = NULL;
+                                        char *command_expanded = NULL;
+                                        const char *const *i;
+                                        int j;
+
+                                        /* remote command */
+                                        command = jt->get_attr( jt, DRMAA_REMOTE_COMMAND );
+                                        if( command == NULL )
+                                                fsd_exc_raise_msg(
+                                                                FSD_DRMAA_ERRNO_CONFLICTING_ATTRIBUTE_VALUES,
+                                                                "drmaa_remote_command not set for job template"
+                                                                );
+
+                                        command_expanded = expand->expand( expand, fsd_strdup(command), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD );
+
+                                        fprintf(fd,"%s", command_expanded); /* we put the commmand at the end of script (instead of using @executable keyword
+                                                                                 in roder to avoid coping of the binary */
+                                        fsd_log_debug(("command = %s\n", command_expanded));
+
+                                        fsd_free(command_expanded);
+
+                                        /* arguments list */
+                                        vector = jt->get_v_attr( jt, DRMAA_V_ARGV );
+
+                                        if( vector )
+                                        {
+                                                fsd_log_debug(("arguments = "));
+
+                                                for( i = vector, j = 2;  *i;  i++, j++ )
+                                                {
+                                                        char *arg_expanded = expand->expand( expand, fsd_strdup(*i), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD );
+
+                                                        fprintf(fd," '%s'", arg_expanded);
+                                                        fsd_log_debug(("%s", arg_expanded));
+
+                                                        fsd_free(arg_expanded);
+                                                }
+                                        }
+
+                                        fprintf(fd," \n");
+                                }
+                                END_TRY
+
+
                         }
                 }