[20] | 1 | /* $Id$ */ |
---|
[1] | 2 | /* |
---|
| 3 | * PSNC DRMAA for LL |
---|
| 4 | * Copyright (C) 2010 Poznan Supercomputing and Networking Center |
---|
| 5 | * |
---|
| 6 | * Licensed under the Apache License, Version 2.0 (the "License"); |
---|
| 7 | * you may not use this file except in compliance with the License. |
---|
| 8 | * You may obtain a copy of the License at |
---|
| 9 | * |
---|
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 11 | * |
---|
| 12 | * Unless required by applicable law or agreed to in writing, software |
---|
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
| 15 | * See the License for the specific language governing permissions and |
---|
| 16 | * limitations under the License. |
---|
| 17 | */ |
---|
| 18 | |
---|
| 19 | #include <string.h> |
---|
| 20 | #include <stdlib.h> |
---|
| 21 | #include <unistd.h> |
---|
| 22 | #include <time.h> |
---|
| 23 | |
---|
| 24 | #include <drmaa_utils/common.h> |
---|
| 25 | #include <drmaa_utils/conf.h> |
---|
| 26 | #include <drmaa_utils/datetime.h> |
---|
| 27 | #include <drmaa_utils/drmaa.h> |
---|
| 28 | #include <drmaa_utils/drmaa_util.h> |
---|
| 29 | #include <drmaa_utils/environ.h> |
---|
| 30 | #include <drmaa_utils/template.h> |
---|
| 31 | |
---|
| 32 | #include <ll_drmaa/job.h> |
---|
| 33 | #include <ll_drmaa/session.h> |
---|
| 34 | #include <ll_drmaa/util.h> |
---|
| 35 | |
---|
| 36 | #include <llapi.h> |
---|
| 37 | |
---|
| 38 | static void |
---|
| 39 | lldrmaa_job_control( fsd_job_t *self, int action ) |
---|
| 40 | { |
---|
| 41 | int rc = 0; |
---|
| 42 | LL_element *errObj = NULL; |
---|
| 43 | LL_terminate_job_info cancel_info; |
---|
| 44 | char *rest = NULL; |
---|
| 45 | char *token = NULL; |
---|
| 46 | char *ptr = NULL; |
---|
| 47 | char *volatile job_list[2]; |
---|
| 48 | |
---|
| 49 | job_list[0] = self->job_id; |
---|
| 50 | job_list[1] = NULL; |
---|
| 51 | |
---|
| 52 | cancel_info.StepId.from_host = NULL; |
---|
| 53 | ptr = fsd_strdup(self->job_id); |
---|
| 54 | fsd_log_enter(( "({job_id=%s}, action=%s)", self->job_id, drmaa_control_to_str(action) )); |
---|
| 55 | |
---|
| 56 | fsd_mutex_lock( &self->session->drm_connection_mutex ); |
---|
| 57 | TRY |
---|
| 58 | { |
---|
| 59 | switch( action ) |
---|
| 60 | { |
---|
| 61 | case DRMAA_CONTROL_SUSPEND: |
---|
| 62 | fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"job::control: could not send %s to job %s. Function not implemented.",drmaa_control_to_str(action), self->job_id); |
---|
| 63 | break; |
---|
| 64 | case DRMAA_CONTROL_HOLD: |
---|
| 65 | rc = ll_control(LL_CONTROL_VERSION, LL_CONTROL_HOLD_USER, NULL, NULL, (char **)job_list, NULL,0); |
---|
| 66 | fsd_log_warning(("This command does not affect a job step that is running unless the job step attempts to enter the Idle state.")); |
---|
| 67 | if(rc) |
---|
| 68 | fsd_exc_raise_fmt(lldrmaa_map_control(rc), |
---|
| 69 | "job::control: could not send %s to job %s. Error code: %d,%s", |
---|
| 70 | drmaa_control_to_str(action), self->job_id, rc, lldrmaa_err_control(rc) |
---|
| 71 | ); |
---|
| 72 | break; |
---|
| 73 | case DRMAA_CONTROL_RESUME: |
---|
| 74 | fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"job::control: could not send %s to job %s. Function not implemented.",drmaa_control_to_str(action), self->job_id); |
---|
| 75 | break; |
---|
| 76 | case DRMAA_CONTROL_RELEASE: |
---|
| 77 | rc = ll_control(LL_CONTROL_VERSION, LL_CONTROL_HOLD_RELEASE, NULL,NULL, (char **)job_list,NULL,0); |
---|
| 78 | if(rc) |
---|
| 79 | fsd_exc_raise_fmt(lldrmaa_map_control(rc), |
---|
| 80 | "job::control: could not send %s to job %s. Error code: %d,%s", |
---|
| 81 | drmaa_control_to_str(action), self->job_id, rc, lldrmaa_err_control(rc) |
---|
| 82 | ); |
---|
| 83 | break; |
---|
| 84 | case DRMAA_CONTROL_TERMINATE: |
---|
| 85 | cancel_info.version_num=LL_PROC_VERSION; |
---|
| 86 | cancel_info.msg=NULL; |
---|
| 87 | |
---|
| 88 | token = strtok_r(ptr, ".", &rest); |
---|
| 89 | if(token == NULL) |
---|
| 90 | { |
---|
| 91 | fsd_exc_raise_fmt(FSD_DRMAA_ERRNO_INVALID_JOB,"Invalid job_id format"); |
---|
| 92 | } |
---|
| 93 | cancel_info.StepId.from_host = fsd_strdup(token); |
---|
| 94 | |
---|
| 95 | token = strtok_r(NULL, ".", &rest); |
---|
| 96 | if(token == NULL) |
---|
| 97 | { |
---|
| 98 | fsd_exc_raise_fmt(FSD_DRMAA_ERRNO_INVALID_JOB,"Invalid job_id format"); |
---|
| 99 | } |
---|
| 100 | |
---|
| 101 | cancel_info.StepId.cluster = fsd_atoi(token); |
---|
| 102 | |
---|
| 103 | token = strtok_r(NULL, ".", &rest); |
---|
| 104 | if(token!=NULL) |
---|
| 105 | { |
---|
| 106 | cancel_info.StepId.proc = fsd_atoi(token); |
---|
| 107 | rc = ll_terminate_job(&cancel_info); |
---|
| 108 | } |
---|
| 109 | else /* No StepId specified */ |
---|
| 110 | { |
---|
| 111 | fsd_exc_raise_fmt(FSD_DRMAA_ERRNO_INVALID_JOB, "Stepid is NULL"); |
---|
| 112 | } |
---|
| 113 | if(rc) |
---|
| 114 | fsd_exc_raise_fmt( |
---|
| 115 | lldrmaa_map_terminate_job(rc), |
---|
| 116 | "job::control: could not send %s to job %s. Error code: %d,%s", |
---|
| 117 | drmaa_control_to_str(action), self->job_id, rc, lldrmaa_err_terminate_job(rc) |
---|
| 118 | ); |
---|
| 119 | |
---|
| 120 | break; |
---|
| 121 | default: |
---|
| 122 | fsd_exc_raise_fmt( |
---|
| 123 | FSD_ERRNO_INVALID_ARGUMENT, |
---|
| 124 | "job::control: unknown action %d", action ); |
---|
| 125 | } |
---|
| 126 | |
---|
| 127 | fsd_log_debug(("job::control: successful")); |
---|
| 128 | } |
---|
| 129 | FINALLY |
---|
| 130 | { |
---|
| 131 | fsd_free(ptr); |
---|
| 132 | if(cancel_info.StepId.from_host!=NULL) |
---|
| 133 | fsd_free(cancel_info.StepId.from_host); |
---|
| 134 | if(errObj != NULL) |
---|
| 135 | { |
---|
| 136 | ll_free_objs(errObj); |
---|
| 137 | ll_deallocate(errObj); |
---|
| 138 | } |
---|
| 139 | |
---|
| 140 | fsd_mutex_unlock( &self->session->drm_connection_mutex ); |
---|
| 141 | } |
---|
| 142 | END_TRY |
---|
| 143 | |
---|
| 144 | fsd_log_return(( "" )); |
---|
| 145 | } |
---|
| 146 | |
---|
| 147 | void |
---|
| 148 | lldrmaa_job_read_job_info( fsd_job_t *self, enum StepState step_state, enum HoldType hold_type ) |
---|
| 149 | { |
---|
| 150 | fsd_log_enter(( "" )); |
---|
| 151 | |
---|
| 152 | fsd_log_debug(("step state: %d",step_state)); |
---|
| 153 | fsd_log_debug(("hold type: %d",hold_type)); |
---|
| 154 | if( hold_type && ( step_state == STATE_HOLD || step_state == STATE_IDLE )) |
---|
| 155 | { |
---|
| 156 | if (hold_type == HOLDTYPE_USER) |
---|
| 157 | self->state = DRMAA_PS_USER_ON_HOLD; |
---|
| 158 | else if (hold_type == HOLDTYPE_SYSTEM) |
---|
| 159 | self->state = DRMAA_PS_SYSTEM_ON_HOLD; |
---|
| 160 | else if (hold_type == HOLDTYPE_USERSYS) |
---|
| 161 | self->state = DRMAA_PS_USER_SYSTEM_ON_HOLD; |
---|
| 162 | else |
---|
| 163 | { |
---|
| 164 | fsd_log_error(("hold_type does not match step_state, %d", hold_type)); |
---|
| 165 | fsd_assert(false); |
---|
| 166 | } |
---|
| 167 | } |
---|
| 168 | else if( step_state == STATE_DEFERRED || step_state == STATE_IDLE || step_state == STATE_PENDING ) |
---|
| 169 | self->state = DRMAA_PS_QUEUED_ACTIVE; |
---|
| 170 | else if( step_state == STATE_RUNNING ) |
---|
| 171 | self->state = DRMAA_PS_RUNNING; |
---|
| 172 | else if( step_state == STATE_PREEMPTED ) |
---|
| 173 | self->state = DRMAA_PS_SYSTEM_SUSPENDED; |
---|
| 174 | else if( step_state == STATE_COMPLETED ) |
---|
| 175 | self->state = DRMAA_PS_DONE; |
---|
| 176 | else if ( step_state == STATE_VACATED) |
---|
| 177 | { |
---|
| 178 | if( ((lldrmaa_session_t*) self->session)->terminate_job_on_vacated ) |
---|
| 179 | { |
---|
| 180 | lldrmaa_job_control(self, DRMAA_CONTROL_TERMINATE); |
---|
| 181 | } |
---|
| 182 | self->state = DRMAA_PS_FAILED; |
---|
| 183 | } |
---|
| 184 | else if( step_state == STATE_CANCELED || step_state == STATE_NOTQUEUED || step_state == STATE_NOTRUN || |
---|
| 185 | step_state == STATE_REJECTED || step_state == STATE_REMOVED || step_state == STATE_TERMINATED || |
---|
| 186 | step_state == STATE_SUBMISSION_ERR) |
---|
| 187 | self->state = DRMAA_PS_FAILED; |
---|
| 188 | else if ( step_state == STATE_COMPLETE_PENDING || step_state == STATE_PREEMPT_PENDING || |
---|
| 189 | step_state == STATE_REJECT_PENDING || step_state == STATE_REMOVE_PENDING || |
---|
| 190 | step_state == STATE_RESUME_PENDING || step_state == STATE_STARTING || step_state == STATE_VACATE_PENDING) |
---|
| 191 | fsd_log_debug(("STATE_X_PENDING")); |
---|
| 192 | else if ( step_state == STATE_UNEXPANDED) |
---|
| 193 | self->state = DRMAA_PS_UNDETERMINED; |
---|
| 194 | else |
---|
| 195 | { |
---|
| 196 | fsd_log_error(("step_state = %d, assert(0)",step_state)); |
---|
| 197 | fsd_assert(false); |
---|
| 198 | } |
---|
| 199 | |
---|
| 200 | self->last_update_time = time(NULL); |
---|
| 201 | |
---|
| 202 | fsd_log_return(( "" )); |
---|
| 203 | } |
---|
| 204 | |
---|
| 205 | void |
---|
| 206 | lldrmaa_job_read_job_info_mon( fsd_job_t *self, const char * state , unsigned status) |
---|
| 207 | { |
---|
| 208 | fsd_log_enter(( "" )); |
---|
| 209 | |
---|
| 210 | self->exit_status = status; |
---|
| 211 | |
---|
| 212 | if( strcmp(state,"JOB_STARTED") == 0) |
---|
| 213 | { |
---|
| 214 | self->state = DRMAA_PS_RUNNING; |
---|
| 215 | } |
---|
| 216 | else if( strcmp(state,"JOB_COMPLETED") == 0) |
---|
| 217 | { |
---|
| 218 | self->state = DRMAA_PS_DONE; |
---|
| 219 | } |
---|
| 220 | else if( strcmp(state,"JOB_VACATED") == 0 ) |
---|
| 221 | { |
---|
| 222 | if( ((lldrmaa_session_t*) self->session)->terminate_job_on_vacated ) |
---|
| 223 | { |
---|
| 224 | lldrmaa_job_control(self, DRMAA_CONTROL_TERMINATE); |
---|
| 225 | } |
---|
| 226 | self->state = DRMAA_PS_FAILED; |
---|
| 227 | } |
---|
| 228 | else if( strcmp(state,"JOB_REJECTED") == 0) |
---|
| 229 | { |
---|
| 230 | self->state = DRMAA_PS_FAILED; |
---|
| 231 | } |
---|
| 232 | else if( strcmp(state,"JOB_REMOVED") == 0) |
---|
| 233 | { |
---|
| 234 | self->state = DRMAA_PS_FAILED; |
---|
| 235 | } |
---|
| 236 | else if( strcmp(state,"JOB_NOTRUN") == 0) |
---|
| 237 | { |
---|
| 238 | self->state = DRMAA_PS_FAILED; |
---|
| 239 | } |
---|
| 240 | else |
---|
| 241 | { |
---|
| 242 | fsd_log_error(("state = %s, assert(0)",state)); |
---|
| 243 | fsd_assert(false); |
---|
| 244 | } |
---|
| 245 | |
---|
| 246 | fsd_log_info(("state: %s -> %s",state, drmaa_job_ps_to_str(self->state))); |
---|
| 247 | |
---|
| 248 | if( self->state >= DRMAA_PS_DONE ) |
---|
| 249 | fsd_cond_broadcast( &self->status_cond ); |
---|
| 250 | |
---|
| 251 | fsd_log_return(( "" )); |
---|
| 252 | } |
---|
| 253 | |
---|
| 254 | static void |
---|
| 255 | lldrmaa_job_update_status( fsd_job_t *self ) |
---|
| 256 | { |
---|
| 257 | enum StepState step_state; |
---|
| 258 | enum HoldType hold_type; |
---|
| 259 | int rc; |
---|
| 260 | int obj_count,err_code; |
---|
| 261 | LL_element * job=NULL,*data=NULL,*step=NULL; |
---|
| 262 | char **host_list = NULL; |
---|
| 263 | lldrmaa_job_t *llself = (lldrmaa_job_t*) self; |
---|
| 264 | |
---|
| 265 | fsd_log_enter(( "({job_id=%s})", self->job_id )); |
---|
| 266 | |
---|
| 267 | fsd_mutex_lock( &self->session->drm_connection_mutex ); |
---|
| 268 | TRY |
---|
| 269 | { |
---|
| 270 | fsd_log_debug(( "drm connection locked" )); |
---|
| 271 | job = ll_query(JOBS); |
---|
| 272 | if (!job) { |
---|
| 273 | fsd_exc_raise_msg(FSD_ERRNO_INTERNAL_ERROR, |
---|
| 274 | "ll_query() returns NULL. The subroutine was unable to create the appropriate pointer."); |
---|
| 275 | } |
---|
| 276 | |
---|
| 277 | fsd_calloc(host_list, 2, char *); |
---|
| 278 | host_list[0] = fsd_strdup(self->job_id); |
---|
| 279 | host_list[1] = NULL; |
---|
| 280 | |
---|
| 281 | fsd_log_debug(("host_list[0]: %s",host_list[0])); |
---|
| 282 | |
---|
| 283 | rc = ll_set_request(job,QUERY_STEPID,host_list,ALL_DATA); |
---|
| 284 | if (rc) { |
---|
| 285 | fsd_exc_raise_fmt(lldrmaa_map_set_request(rc), |
---|
| 286 | "ll_set_request() return code is non-zero. %s",lldrmaa_err_set_request(rc)); |
---|
| 287 | } |
---|
| 288 | |
---|
| 289 | data = ll_get_objs(job, LL_SCHEDD, NULL, &obj_count, &err_code); |
---|
| 290 | if (data == NULL) { |
---|
| 291 | fsd_log_debug(("Code: %d ll_get_objs() returns NULL. %s",lldrmaa_map_get_objs(err_code), lldrmaa_err_get_objs(err_code) )); |
---|
| 292 | /* This error means that there is no info in LL_SCHEDD but this job have probably ended and will be detected by wait_thread. This won't be fsd_log_warning because displays too often during program execution */ |
---|
| 293 | } else { |
---|
| 294 | |
---|
| 295 | rc = ll_get_data(data, LL_JobGetFirstStep, &step); |
---|
| 296 | if(rc) |
---|
| 297 | { |
---|
| 298 | fsd_exc_raise_fmt(lldrmaa_map_get_data(rc), |
---|
| 299 | "ll_get_data() return code is non-zero. %s",lldrmaa_err_get_data(rc)); |
---|
| 300 | } |
---|
| 301 | |
---|
| 302 | rc = ll_get_data(step, LL_StepState,&step_state); |
---|
| 303 | if (rc) { |
---|
| 304 | fsd_exc_raise_fmt(lldrmaa_map_get_data(rc), |
---|
| 305 | "ll_get_data() return code is non-zero. %s",lldrmaa_err_get_data(rc)); |
---|
| 306 | } |
---|
| 307 | |
---|
| 308 | rc = ll_get_data(step, LL_StepHoldType, &hold_type); |
---|
| 309 | if (rc) { |
---|
| 310 | fsd_exc_raise_fmt(lldrmaa_map_get_data(rc), |
---|
| 311 | "ll_get_data() return code is non-zero. %s",lldrmaa_err_get_data(rc)); |
---|
| 312 | } |
---|
| 313 | |
---|
| 314 | llself->read_job_info( self, step_state, hold_type); |
---|
| 315 | |
---|
| 316 | fsd_log_info(("LL State: %d -> DRMAA: %s",step_state,drmaa_job_ps_to_str(self->state))); |
---|
| 317 | } |
---|
| 318 | } |
---|
| 319 | FINALLY |
---|
| 320 | { |
---|
| 321 | if(job!=NULL) |
---|
| 322 | { |
---|
| 323 | ll_free_objs(job); |
---|
| 324 | ll_deallocate(job); |
---|
| 325 | } |
---|
| 326 | if(data!=NULL) |
---|
| 327 | { |
---|
| 328 | ll_free_objs(data); |
---|
| 329 | ll_deallocate(data); |
---|
| 330 | } |
---|
| 331 | if(step!=NULL) |
---|
| 332 | { |
---|
| 333 | ll_free_objs(step); |
---|
| 334 | ll_deallocate(step); |
---|
| 335 | } |
---|
| 336 | if(host_list!=NULL) |
---|
| 337 | fsd_free_vector (host_list); |
---|
| 338 | |
---|
| 339 | fsd_mutex_unlock( &self->session->drm_connection_mutex ); |
---|
| 340 | } |
---|
| 341 | END_TRY |
---|
| 342 | |
---|
| 343 | fsd_log_return(( "" )); |
---|
| 344 | } |
---|
| 345 | |
---|
| 346 | fsd_job_t * |
---|
| 347 | lldrmaa_job_new( char *job_id ) |
---|
| 348 | { |
---|
| 349 | lldrmaa_job_t *self = NULL; |
---|
| 350 | self = (lldrmaa_job_t*)fsd_job_new( job_id ); |
---|
| 351 | fsd_realloc( self, 1, lldrmaa_job_t ); |
---|
| 352 | self->super.control = lldrmaa_job_control; |
---|
| 353 | self->super.update_status = lldrmaa_job_update_status; |
---|
| 354 | self->read_job_info_mon = lldrmaa_job_read_job_info_mon; |
---|
| 355 | self->read_job_info = lldrmaa_job_read_job_info; |
---|
| 356 | return (fsd_job_t*)self; |
---|
| 357 | } |
---|
| 358 | |
---|
| 359 | char * |
---|
| 360 | lldrmaa_job_create_req( |
---|
| 361 | fsd_drmaa_session_t *session, |
---|
| 362 | const fsd_template_t *jt, |
---|
| 363 | fsd_environ_t **envp, |
---|
| 364 | unsigned bulk, |
---|
| 365 | int start, |
---|
| 366 | int incr |
---|
| 367 | ) |
---|
| 368 | { |
---|
| 369 | char *volatile cmd_file = NULL; |
---|
| 370 | fsd_expand_drmaa_ph_t *volatile expand = NULL; |
---|
| 371 | TRY |
---|
| 372 | { |
---|
| 373 | expand = fsd_expand_drmaa_ph_new( NULL, NULL, fsd_strdup("0") ); |
---|
| 374 | cmd_file = (char *) lldrmaa_job_create_command_file( session, jt, envp, expand, bulk, start, incr ); |
---|
| 375 | } |
---|
| 376 | EXCEPT_DEFAULT |
---|
| 377 | { |
---|
| 378 | remove(cmd_file); |
---|
| 379 | fsd_exc_reraise(); |
---|
| 380 | } |
---|
| 381 | FINALLY |
---|
| 382 | { |
---|
| 383 | if( expand ) |
---|
| 384 | expand->destroy( expand ); |
---|
| 385 | } |
---|
| 386 | END_TRY |
---|
| 387 | return cmd_file; |
---|
| 388 | } |
---|
| 389 | |
---|
| 390 | |
---|
| 391 | static char * |
---|
| 392 | internal_map_file( fsd_expand_drmaa_ph_t *expand, const char *path, |
---|
| 393 | bool *host_given, const char *name ) |
---|
| 394 | { |
---|
| 395 | const char *p; |
---|
| 396 | for( p = path; *p != ':'; p++ ) |
---|
| 397 | if( *p == '\0' ) |
---|
| 398 | fsd_exc_raise_fmt( FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_FORMAT, |
---|
| 399 | "invalid format of drmaa_%s_path: missing colon", name ); |
---|
| 400 | if( host_given ) |
---|
| 401 | *host_given = ( p != path ); |
---|
| 402 | p++; |
---|
| 403 | return expand->expand( expand, fsd_strdup(p), |
---|
| 404 | FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD | FSD_DRMAA_PH_INCR ); |
---|
| 405 | } |
---|
| 406 | |
---|
| 407 | char * |
---|
| 408 | lldrmaa_job_create_command_file( |
---|
| 409 | fsd_drmaa_session_t *session, |
---|
| 410 | const fsd_template_t *jt, |
---|
| 411 | fsd_environ_t **envp, |
---|
| 412 | fsd_expand_drmaa_ph_t *expand, |
---|
| 413 | unsigned n_jobs, |
---|
| 414 | int start, |
---|
| 415 | int incr |
---|
| 416 | ) |
---|
| 417 | { |
---|
| 418 | char template_cmd[] = "/tmp/drmaa_cmd_XXXXXX"; |
---|
| 419 | int fdt; |
---|
| 420 | FILE * fd; |
---|
| 421 | |
---|
| 422 | |
---|
| 423 | if((fdt = mkstemp(template_cmd) ) == -1) |
---|
| 424 | { |
---|
| 425 | fsd_log_error(("Can't create cmd file")); |
---|
| 426 | fsd_exc_raise_msg(FSD_ERRNO_INTERNAL_ERROR,"Can't create cmd file"); |
---|
| 427 | } |
---|
| 428 | close(fdt); |
---|
| 429 | |
---|
| 430 | if((fd = fopen(template_cmd,"w") ) != NULL ) |
---|
| 431 | { |
---|
| 432 | const char *input_path_orig = NULL; |
---|
| 433 | const char *output_path_orig = NULL; |
---|
| 434 | const char *error_path_orig = NULL; |
---|
| 435 | char *volatile input_path = NULL; |
---|
| 436 | char *volatile output_path = NULL; |
---|
| 437 | char *volatile error_path = NULL; |
---|
| 438 | bool input_host = false; |
---|
| 439 | bool output_host = false; |
---|
| 440 | bool error_host = false; |
---|
| 441 | bool join_files = false; |
---|
| 442 | const char *value; |
---|
| 443 | const char *const *vector; |
---|
| 444 | const char *job_category = "default"; |
---|
| 445 | fsd_log_debug(("Cmd file: %s opened",template_cmd)); |
---|
| 446 | fprintf(fd,"# File created by LoadLeveler DRMAA. If no LL DRMAA applications are running you can delete this file\n"); |
---|
| 447 | |
---|
| 448 | /* job name */ |
---|
| 449 | value = jt->get_attr( jt, DRMAA_JOB_NAME ); |
---|
| 450 | if( value ) |
---|
| 451 | { |
---|
| 452 | fprintf(fd,"# @ job_name = %s\n",value); |
---|
| 453 | fsd_log_debug(("# @ job_name = %s",value)); |
---|
| 454 | } |
---|
| 455 | |
---|
| 456 | /* job state at submit */ |
---|
| 457 | value = jt->get_attr( jt, DRMAA_JS_STATE ); |
---|
| 458 | if( value ) |
---|
| 459 | { |
---|
| 460 | if( 0 == strcmp( value, DRMAA_SUBMISSION_STATE_ACTIVE ) ) |
---|
| 461 | {} |
---|
| 462 | else if( 0 == strcmp( value, DRMAA_SUBMISSION_STATE_HOLD ) ) |
---|
| 463 | { |
---|
| 464 | fprintf(fd,"# @ hold = user\n"); |
---|
| 465 | fsd_log_debug(("# @ hold = user")); |
---|
| 466 | } |
---|
| 467 | else |
---|
| 468 | fsd_exc_raise_msg( |
---|
| 469 | FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, |
---|
| 470 | "invalid value of drmaa_js_state attribute" ); |
---|
| 471 | } |
---|
| 472 | |
---|
| 473 | TRY |
---|
| 474 | { |
---|
| 475 | const char *command = NULL; |
---|
| 476 | char *command_expanded = NULL; |
---|
| 477 | const char *const *i; |
---|
| 478 | int j; |
---|
| 479 | |
---|
| 480 | /* remote command */ |
---|
| 481 | command = jt->get_attr( jt, DRMAA_REMOTE_COMMAND ); |
---|
| 482 | if( command == NULL ) |
---|
| 483 | fsd_exc_raise_msg( |
---|
| 484 | FSD_DRMAA_ERRNO_CONFLICTING_ATTRIBUTE_VALUES, |
---|
| 485 | "drmaa_remote_command not set for job template" |
---|
| 486 | ); |
---|
| 487 | |
---|
| 488 | command_expanded = expand->expand( expand, fsd_strdup(command), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD ); |
---|
| 489 | |
---|
| 490 | fprintf(fd,"# @ executable = %s\n", command_expanded); |
---|
| 491 | fsd_log_debug(("# @ executable = %s\n", command_expanded)); |
---|
| 492 | |
---|
| 493 | fsd_free(command_expanded); |
---|
| 494 | |
---|
| 495 | /* arguments list */ |
---|
| 496 | vector = jt->get_v_attr( jt, DRMAA_V_ARGV ); |
---|
| 497 | |
---|
| 498 | if( vector ) |
---|
| 499 | { |
---|
| 500 | fprintf(fd,"# @ arguments ="); |
---|
| 501 | fsd_log_debug(("# @ arguments =")); |
---|
| 502 | |
---|
| 503 | for( i = vector, j = 2; *i; i++, j++ ) |
---|
| 504 | { |
---|
| 505 | char *arg_expanded = expand->expand( expand, fsd_strdup(*i), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD ); |
---|
| 506 | |
---|
| 507 | fprintf(fd," '%s'", arg_expanded); |
---|
| 508 | fsd_log_debug(("%s", arg_expanded)); |
---|
| 509 | |
---|
| 510 | fsd_free(arg_expanded); |
---|
| 511 | } |
---|
| 512 | } |
---|
| 513 | |
---|
| 514 | fprintf(fd," \n"); |
---|
| 515 | } |
---|
| 516 | END_TRY |
---|
| 517 | |
---|
| 518 | /* start time */ |
---|
| 519 | value = jt->get_attr( jt, DRMAA_START_TIME ); |
---|
| 520 | if( value ) |
---|
| 521 | { |
---|
| 522 | char buf[30]; |
---|
| 523 | const time_t time_ll = fsd_datetime_parse( value ); |
---|
| 524 | struct tm temp; |
---|
| 525 | localtime_r(&time_ll,&temp); |
---|
| 526 | strftime(buf, sizeof(buf),"%m/%d/%Y %H:%M:%S", &temp ); |
---|
| 527 | fprintf(fd,"# @ startdate = %s\n", buf); |
---|
| 528 | fsd_log_debug(("# @ startdate = %s", buf)); |
---|
| 529 | fsd_log_debug(( "\n drmaa_start_time: %s -> %ld", value, (long)time_ll )); |
---|
| 530 | } |
---|
| 531 | |
---|
| 532 | /* environment */ |
---|
| 533 | vector = jt->get_v_attr( jt, DRMAA_V_ENV ); |
---|
| 534 | if( vector ) |
---|
| 535 | { |
---|
| 536 | const char *const *i; |
---|
| 537 | |
---|
| 538 | fprintf(fd,"# @ environment ="); |
---|
| 539 | fsd_log_debug(("# @ environment =")); |
---|
| 540 | for( i = vector; *i; i++ ) |
---|
| 541 | { |
---|
| 542 | fprintf(fd," %s;",*i); |
---|
| 543 | fsd_log_debug((" %s;",*i)); |
---|
| 544 | } |
---|
| 545 | fprintf(fd," \n"); |
---|
| 546 | } |
---|
| 547 | |
---|
| 548 | /* wall clock time soft and hard limit */ |
---|
| 549 | { |
---|
| 550 | const char *value2; |
---|
| 551 | value = jt->get_attr( jt, DRMAA_WCT_SLIMIT ); |
---|
| 552 | value2 = jt->get_attr( jt, DRMAA_WCT_HLIMIT ); |
---|
| 553 | if( value && value2) |
---|
| 554 | { |
---|
| 555 | fprintf(fd,"# @ wall_clock_limit = %s,%s\n",value,value2); |
---|
| 556 | fsd_log_debug(("# @ wall_clock_limit = %s,%s",value,value2)); |
---|
| 557 | } |
---|
| 558 | else if (value) |
---|
| 559 | { |
---|
| 560 | fprintf(fd,"# @ wall_clock_limit = %s\n",value); |
---|
| 561 | fsd_log_debug(("# @ wall_clock_limit = %s",value)); |
---|
| 562 | } |
---|
| 563 | else if (value2) |
---|
| 564 | { |
---|
| 565 | fprintf(fd,"# @ wall_clock_limit = ,%s\n",value); |
---|
| 566 | fsd_log_debug(("# @ wall_clock_limit = ,%s",value)); |
---|
| 567 | } |
---|
| 568 | } |
---|
| 569 | |
---|
| 570 | /* native specification */ |
---|
| 571 | value = jt->get_attr( jt, DRMAA_NATIVE_SPECIFICATION ); |
---|
| 572 | if( value ) |
---|
| 573 | { |
---|
| 574 | char * temp = fsd_strdup(value); |
---|
| 575 | temp = fsd_replace(temp,"@","\n# $"); /* infinite loop when try replace @ with # @ */ |
---|
| 576 | temp = fsd_replace(temp,"# $","# @ "); |
---|
| 577 | |
---|
| 578 | fprintf(fd,"# Native specification:%s\n", temp); |
---|
| 579 | fsd_log_debug(("Native specification:%s", temp)); |
---|
| 580 | fsd_free(temp); |
---|
| 581 | } |
---|
| 582 | |
---|
| 583 | /* job category */ |
---|
| 584 | value = jt->get_attr( jt, DRMAA_JOB_CATEGORY ); |
---|
| 585 | if( value ) |
---|
| 586 | job_category = value; |
---|
| 587 | |
---|
| 588 | { |
---|
| 589 | char * temp; |
---|
| 590 | fsd_conf_option_t *category_value = NULL; |
---|
| 591 | category_value = fsd_conf_dict_get( session->job_categories, job_category ); |
---|
| 592 | if( category_value != NULL ) |
---|
| 593 | { |
---|
| 594 | if( category_value->type != FSD_CONF_STRING ) |
---|
| 595 | fsd_exc_raise_fmt( |
---|
| 596 | FSD_ERRNO_INTERNAL_ERROR, |
---|
| 597 | "configuration error: job category should be string" |
---|
| 598 | ); |
---|
| 599 | |
---|
| 600 | temp = fsd_strdup(category_value->val.string); |
---|
| 601 | temp = fsd_replace(temp,"@","\n# $"); /* infinite loop when try replace @ with # @ */ |
---|
| 602 | temp = fsd_replace(temp,"# $","# @ "); |
---|
| 603 | |
---|
| 604 | fprintf(fd,"# Job category:%s\n", temp); |
---|
| 605 | fsd_log_debug(("Job category:\n%s",temp)); |
---|
| 606 | fsd_free(temp); |
---|
| 607 | } |
---|
| 608 | else |
---|
| 609 | { |
---|
| 610 | if( value != NULL ) |
---|
| 611 | fsd_exc_raise_fmt( |
---|
| 612 | FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, |
---|
| 613 | "invalid job category: %s", job_category |
---|
| 614 | ); |
---|
| 615 | } |
---|
| 616 | } |
---|
| 617 | |
---|
| 618 | |
---|
| 619 | { |
---|
| 620 | unsigned i; |
---|
| 621 | for(i=0;i<n_jobs;i++) |
---|
| 622 | { |
---|
| 623 | expand->set(expand, FSD_DRMAA_PH_INCR,fsd_asprintf("%d", start+incr*i)); /* set current value */ |
---|
| 624 | |
---|
| 625 | /* job working directory */ |
---|
| 626 | value = jt->get_attr( jt, DRMAA_WD ); |
---|
| 627 | if( value ) |
---|
| 628 | { |
---|
| 629 | char *cwd_expanded = expand->expand( expand, fsd_strdup(value), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_INCR ); |
---|
| 630 | |
---|
| 631 | expand->set( expand, FSD_DRMAA_PH_WD, fsd_strdup(cwd_expanded)); |
---|
| 632 | fprintf(fd,"# @ initialdir = %s\n",cwd_expanded); |
---|
| 633 | fsd_log_debug(("# @ initialdir = %s",cwd_expanded)); |
---|
| 634 | |
---|
| 635 | fsd_free(cwd_expanded); |
---|
| 636 | } |
---|
| 637 | |
---|
| 638 | TRY |
---|
| 639 | { |
---|
| 640 | /* input path */ |
---|
| 641 | input_path_orig = jt->get_attr( jt, DRMAA_INPUT_PATH ); |
---|
| 642 | if( input_path_orig ) |
---|
| 643 | { |
---|
| 644 | input_path = internal_map_file( expand, input_path_orig, &input_host, |
---|
| 645 | "input" ); |
---|
| 646 | fsd_log_debug(( "\n drmaa_input_path: %s -> %s", |
---|
| 647 | input_path_orig, input_path )); |
---|
| 648 | } |
---|
| 649 | |
---|
| 650 | /* output path */ |
---|
| 651 | output_path_orig = jt->get_attr( jt, DRMAA_OUTPUT_PATH ); |
---|
| 652 | if( output_path_orig ) |
---|
| 653 | { |
---|
| 654 | output_path = internal_map_file( expand, output_path_orig, &output_host, |
---|
| 655 | "output" ); |
---|
| 656 | fsd_log_debug(( "\n drmaa_output_path: %s -> %s", |
---|
| 657 | output_path_orig, output_path )); |
---|
| 658 | } |
---|
| 659 | |
---|
| 660 | /* error path */ |
---|
| 661 | error_path_orig = jt->get_attr( jt, DRMAA_ERROR_PATH ); |
---|
| 662 | if( error_path_orig ) |
---|
| 663 | { |
---|
| 664 | error_path = internal_map_file( expand, error_path_orig, &error_host, |
---|
| 665 | "error" ); |
---|
| 666 | fsd_log_debug(( "\n drmaa_error_path: %s -> %s", |
---|
| 667 | error_path_orig, error_path )); |
---|
| 668 | } |
---|
| 669 | |
---|
| 670 | /* join files */ |
---|
| 671 | value = jt->get_attr( jt, DRMAA_JOIN_FILES ); |
---|
| 672 | if( value ) |
---|
| 673 | { |
---|
| 674 | if( (value[0] == 'y' || value[0] == 'Y') && value[1] == '\0' ) |
---|
| 675 | join_files = true; |
---|
| 676 | else if( (value[0] == 'n' || value[0] == 'N') && value[1] == '\0' ) |
---|
| 677 | join_files = false; |
---|
| 678 | else |
---|
| 679 | fsd_exc_raise_msg( |
---|
| 680 | FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, |
---|
| 681 | "invalid value of drmaa_join_files attribute" ); |
---|
| 682 | } |
---|
| 683 | |
---|
| 684 | if( join_files ) |
---|
| 685 | { |
---|
| 686 | if( output_path == NULL ) |
---|
| 687 | fsd_exc_raise_msg( |
---|
| 688 | FSD_DRMAA_ERRNO_CONFLICTING_ATTRIBUTE_VALUES, |
---|
| 689 | "drmaa_join_files is set and output file is not given" ); |
---|
| 690 | if( error_path!=NULL && 0 != strcmp( output_path, error_path ) ) |
---|
| 691 | fsd_log_warning(( "Error file was given but will be ignored " |
---|
| 692 | "since drmaa_join_files was set." )); |
---|
| 693 | |
---|
| 694 | if (error_path) |
---|
| 695 | fsd_free(error_path); |
---|
| 696 | |
---|
| 697 | error_path = fsd_strdup(output_path); |
---|
| 698 | } |
---|
| 699 | else |
---|
| 700 | { |
---|
| 701 | if( error_path == NULL && output_path ) |
---|
| 702 | error_path = fsd_strdup( "/dev/null" ); |
---|
| 703 | if( output_path == NULL && error_path ) |
---|
| 704 | output_path = fsd_strdup( "/dev/null" ); |
---|
| 705 | } |
---|
| 706 | |
---|
| 707 | |
---|
| 708 | /* email addresses to send notifications */ |
---|
| 709 | vector = jt->get_v_attr( jt, DRMAA_V_EMAIL ); |
---|
| 710 | if( vector && vector[0] ) |
---|
| 711 | { |
---|
| 712 | /* only to one email address message may be send */ |
---|
| 713 | fprintf(fd,"# @ notify_user = %s\n",vector[0]); |
---|
| 714 | fsd_log_debug(("# @ notify_user = %s\n",vector[0])); |
---|
| 715 | if( vector[1] != NULL ) |
---|
| 716 | { |
---|
| 717 | fsd_log_error(( "LL only supports one e-mail " |
---|
| 718 | "notification address" )); |
---|
| 719 | fsd_exc_raise_msg(FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE,"LL only supports one e-mail " |
---|
| 720 | "notification address"); |
---|
| 721 | } |
---|
| 722 | } |
---|
| 723 | |
---|
| 724 | /* block email */ |
---|
| 725 | value = jt->get_attr( jt, DRMAA_BLOCK_EMAIL ); |
---|
| 726 | if( value ) |
---|
| 727 | { |
---|
| 728 | bool block; |
---|
| 729 | if( strcmp(value, "0") == 0 ) |
---|
| 730 | { |
---|
| 731 | block = true; |
---|
| 732 | fprintf(fd,"# @ notification = never\n"); |
---|
| 733 | fsd_log_debug(("# @ notification = never")); |
---|
| 734 | } |
---|
| 735 | else if( strcmp(value, "1") == 0 ) |
---|
| 736 | block = false; |
---|
| 737 | else |
---|
| 738 | fsd_exc_raise_msg( |
---|
| 739 | FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, |
---|
| 740 | "invalid value of drmaa_block_email attribute" ); |
---|
| 741 | |
---|
| 742 | if( block && output_path == NULL ) |
---|
| 743 | { |
---|
| 744 | fsd_log_debug(( "output path not set and we want to block e-mail, " |
---|
| 745 | "set to /dev/null" )); |
---|
| 746 | output_path = fsd_strdup( "/dev/null" ); |
---|
| 747 | } |
---|
| 748 | } |
---|
| 749 | |
---|
| 750 | if( input_path ) |
---|
| 751 | { |
---|
| 752 | fprintf(fd,"# @ input = %s\n", input_path); |
---|
| 753 | fsd_log_debug(("# @ input = %s", input_path)); |
---|
| 754 | } |
---|
| 755 | |
---|
| 756 | if( output_path ) |
---|
| 757 | { |
---|
| 758 | fprintf(fd,"# @ output = %s\n", output_path); |
---|
| 759 | fsd_log_debug(("# @ output = %s", output_path)); |
---|
| 760 | } |
---|
| 761 | |
---|
| 762 | if( error_path ) |
---|
| 763 | { |
---|
| 764 | fprintf(fd,"# @ error = %s\n", error_path); |
---|
| 765 | fsd_log_debug(("# @ error = %s", error_path)); |
---|
| 766 | } |
---|
| 767 | } |
---|
| 768 | FINALLY |
---|
| 769 | { |
---|
| 770 | fsd_free( input_path ); |
---|
| 771 | fsd_free( output_path ); |
---|
| 772 | fsd_free( error_path ); |
---|
| 773 | input_path = NULL; |
---|
| 774 | output_path = NULL; |
---|
| 775 | error_path = NULL; |
---|
| 776 | } |
---|
| 777 | END_TRY |
---|
| 778 | |
---|
| 779 | fprintf(fd,"# @ queue\n"); |
---|
| 780 | fsd_log_debug(("# @ queue")); |
---|
| 781 | } |
---|
| 782 | } |
---|
| 783 | |
---|
| 784 | fclose(fd); |
---|
| 785 | } |
---|
| 786 | else |
---|
| 787 | { |
---|
| 788 | fsd_log_error(("Can't create cmd file")); |
---|
| 789 | fsd_exc_raise_msg(FSD_ERRNO_INTERNAL_ERROR,"Can't create cmd file"); |
---|
| 790 | } |
---|
| 791 | |
---|
| 792 | return fsd_strdup(template_cmd); |
---|
| 793 | } |
---|
| 794 | |
---|