/* $Id$ */ /* * PSNC DRMAA for LL * Copyright (C) 2010 Poznan Supercomputing and Networking Center * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LL_DRMAA_MAX_MISSING_TIME (60) static void lldrmaa_job_control( fsd_job_t *self, int action ) { int rc = 0; LL_element *errObj = NULL; LL_terminate_job_info cancel_info; char *rest = NULL; char *token = NULL; char *ptr = NULL; char *volatile job_list[2]; job_list[0] = self->job_id; job_list[1] = NULL; cancel_info.StepId.from_host = NULL; ptr = fsd_strdup(self->job_id); fsd_log_enter(( "({job_id=%s}, action=%s)", self->job_id, drmaa_control_to_str(action) )); fsd_mutex_lock( &self->session->drm_connection_mutex ); TRY { switch( action ) { case DRMAA_CONTROL_SUSPEND: fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"job::control: could not send %s to job %s. Function not implemented.",drmaa_control_to_str(action), self->job_id); break; case DRMAA_CONTROL_HOLD: rc = ll_control(LL_CONTROL_VERSION, LL_CONTROL_HOLD_USER, NULL, NULL, (char **)job_list, NULL,0); fsd_log_warning(("This command does not affect a job step that is running unless the job step attempts to enter the Idle state.")); if(rc) fsd_exc_raise_fmt(lldrmaa_map_control(rc), "job::control: could not send %s to job %s. Error code: %d,%s", drmaa_control_to_str(action), self->job_id, rc, lldrmaa_err_control(rc) ); break; case DRMAA_CONTROL_RESUME: fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"job::control: could not send %s to job %s. Function not implemented.",drmaa_control_to_str(action), self->job_id); break; case DRMAA_CONTROL_RELEASE: rc = ll_control(LL_CONTROL_VERSION, LL_CONTROL_HOLD_RELEASE, NULL,NULL, (char **)job_list,NULL,0); if(rc) fsd_exc_raise_fmt(lldrmaa_map_control(rc), "job::control: could not send %s to job %s. Error code: %d,%s", drmaa_control_to_str(action), self->job_id, rc, lldrmaa_err_control(rc) ); break; case DRMAA_CONTROL_TERMINATE: cancel_info.version_num=LL_PROC_VERSION; cancel_info.msg=NULL; token = strtok_r(ptr, ".", &rest); if(token == NULL) { fsd_exc_raise_fmt(FSD_DRMAA_ERRNO_INVALID_JOB,"Invalid job_id format"); } cancel_info.StepId.from_host = fsd_strdup(token); token = strtok_r(NULL, ".", &rest); if(token == NULL) { fsd_exc_raise_fmt(FSD_DRMAA_ERRNO_INVALID_JOB,"Invalid job_id format"); } cancel_info.StepId.cluster = fsd_atoi(token); token = strtok_r(NULL, ".", &rest); if(token!=NULL) { cancel_info.StepId.proc = fsd_atoi(token); rc = ll_terminate_job(&cancel_info); } else /* No StepId specified */ { fsd_exc_raise_fmt(FSD_DRMAA_ERRNO_INVALID_JOB, "Stepid is NULL"); } if(rc) fsd_exc_raise_fmt( lldrmaa_map_terminate_job(rc), "job::control: could not send %s to job %s. Error code: %d,%s", drmaa_control_to_str(action), self->job_id, rc, lldrmaa_err_terminate_job(rc) ); break; default: fsd_exc_raise_fmt( FSD_ERRNO_INVALID_ARGUMENT, "job::control: unknown action %d", action ); } fsd_log_debug(("job::control: successful")); } FINALLY { fsd_free(ptr); if(cancel_info.StepId.from_host!=NULL) fsd_free(cancel_info.StepId.from_host); if(errObj != NULL) { ll_free_objs(errObj); ll_deallocate(errObj); } fsd_mutex_unlock( &self->session->drm_connection_mutex ); } END_TRY fsd_log_return(( "" )); } void lldrmaa_job_read_job_info( fsd_job_t *self, enum StepState step_state, enum HoldType hold_type ) { fsd_log_enter(( "" )); fsd_log_debug(("step state: %d",step_state)); fsd_log_debug(("hold type: %d",hold_type)); if( hold_type && ( step_state == STATE_HOLD || step_state == STATE_IDLE )) { if (hold_type == HOLDTYPE_USER) self->state = DRMAA_PS_USER_ON_HOLD; else if (hold_type == HOLDTYPE_SYSTEM) self->state = DRMAA_PS_SYSTEM_ON_HOLD; else if (hold_type == HOLDTYPE_USERSYS) self->state = DRMAA_PS_USER_SYSTEM_ON_HOLD; else { fsd_log_error(("hold_type does not match step_state, %d", hold_type)); fsd_assert(false); } } else if( step_state == STATE_DEFERRED || step_state == STATE_IDLE || step_state == STATE_PENDING ) self->state = DRMAA_PS_QUEUED_ACTIVE; else if( step_state == STATE_RUNNING ) self->state = DRMAA_PS_RUNNING; else if( step_state == STATE_PREEMPTED ) self->state = DRMAA_PS_SYSTEM_SUSPENDED; else if( step_state == STATE_COMPLETED ) self->state = DRMAA_PS_DONE; else if ( step_state == STATE_VACATED) { if( ((lldrmaa_session_t*) self->session)->terminate_job_on_vacated ) { lldrmaa_job_control(self, DRMAA_CONTROL_TERMINATE); } self->state = DRMAA_PS_FAILED; } else if( step_state == STATE_CANCELED || step_state == STATE_NOTQUEUED || step_state == STATE_NOTRUN || step_state == STATE_REJECTED || step_state == STATE_REMOVED || step_state == STATE_TERMINATED || step_state == STATE_SUBMISSION_ERR) self->state = DRMAA_PS_FAILED; else if ( step_state == STATE_COMPLETE_PENDING || step_state == STATE_PREEMPT_PENDING || step_state == STATE_REJECT_PENDING || step_state == STATE_REMOVE_PENDING || step_state == STATE_RESUME_PENDING || step_state == STATE_STARTING || step_state == STATE_VACATE_PENDING) fsd_log_debug(("STATE_X_PENDING")); else if ( step_state == STATE_UNEXPANDED) self->state = DRMAA_PS_UNDETERMINED; else { fsd_log_error(("step_state = %d, assert(0)",step_state)); fsd_assert(false); } self->last_update_time = time(NULL); if( self->state >= DRMAA_PS_DONE ) fsd_cond_broadcast( &self->status_cond ); fsd_log_return(( "" )); } void lldrmaa_job_read_job_info_mon( fsd_job_t *self, const char * state , unsigned status) { fsd_log_enter(( "" )); self->exit_status = status; if( strcmp(state,"JOB_STARTED") == 0) { self->state = DRMAA_PS_RUNNING; } else if( strcmp(state,"JOB_COMPLETED") == 0) { self->state = DRMAA_PS_DONE; } else if( strcmp(state,"JOB_VACATED") == 0 ) { if( ((lldrmaa_session_t*) self->session)->terminate_job_on_vacated ) { lldrmaa_job_control(self, DRMAA_CONTROL_TERMINATE); } self->state = DRMAA_PS_FAILED; } else if( strcmp(state,"JOB_REJECTED") == 0) { self->state = DRMAA_PS_FAILED; } else if( strcmp(state,"JOB_REMOVED") == 0) { self->state = DRMAA_PS_FAILED; } else if( strcmp(state,"JOB_NOTRUN") == 0) { self->state = DRMAA_PS_FAILED; } else { fsd_log_error(("state = %s, assert(0)",state)); fsd_assert(false); } fsd_log_info(("state: %s -> %s",state, drmaa_job_ps_to_str(self->state))); if( self->state >= DRMAA_PS_DONE ) fsd_cond_broadcast( &self->status_cond ); fsd_log_return(( "" )); } static void lldrmaa_job_update_status( fsd_job_t *self ) { enum StepState step_state; enum HoldType hold_type; int rc; int obj_count,err_code; LL_element * job=NULL,*data=NULL,*step=NULL; char **host_list = NULL; lldrmaa_job_t *llself = (lldrmaa_job_t*) self; fsd_log_enter(( "({job_id=%s})", self->job_id )); fsd_mutex_lock( &self->session->drm_connection_mutex ); TRY { fsd_log_debug(( "drm connection locked" )); job = ll_query(JOBS); if (!job) { fsd_exc_raise_msg(FSD_ERRNO_INTERNAL_ERROR, "ll_query() returns NULL. The subroutine was unable to create the appropriate pointer."); } fsd_calloc(host_list, 2, char *); host_list[0] = fsd_strdup(self->job_id); host_list[1] = NULL; fsd_log_debug(("host_list[0]: %s",host_list[0])); rc = ll_set_request(job,QUERY_STEPID,host_list,ALL_DATA); if (rc) { fsd_exc_raise_fmt(lldrmaa_map_set_request(rc), "ll_set_request() return code is non-zero. %s",lldrmaa_err_set_request(rc)); } data = ll_get_objs(job, LL_CM, NULL, &obj_count, &err_code); if (data == NULL) { if (err_code == LL_GET_OBJS_NO_OBJECTS_ERR) { if (self->state != DRMAA_PS_UNDETERMINED) { fsd_log_info(("Job %s missing. Assuming finished", self->job_id)); self->state = DRMAA_PS_DONE; self->exit_status = 0; fsd_cond_broadcast( &self->status_cond ); } else if (llself->missing_time == 0) { llself->missing_time = time(NULL); fsd_log_debug(("Job %s missing for the first time", self->job_id)); /* Job may not yet be visible in LL */ fsd_cond_broadcast( &self->status_cond ); } else if (time(NULL) - llself->missing_time > LL_DRMAA_MAX_MISSING_TIME) { fsd_log_error(("Job %s missing for more then %d seconds. Assuming failed", self->job_id, LL_DRMAA_MAX_MISSING_TIME)); /* Job may not yet be visible in LL */ self->state = DRMAA_PS_FAILED; self->exit_status = -1; } else { fsd_log_debug(("Job %s still missing", self->job_id)); } } else { fsd_log_error(("Code: %d,%d ll_get_objs() returns NULL. %s", err_code, lldrmaa_map_get_objs(err_code), lldrmaa_err_get_objs(err_code) )); } } else { rc = ll_get_data(data, LL_JobGetFirstStep, &step); if(rc) { fsd_exc_raise_fmt(lldrmaa_map_get_data(rc), "ll_get_data() return code is non-zero. %s",lldrmaa_err_get_data(rc)); } rc = ll_get_data(step, LL_StepState,&step_state); if (rc) { fsd_exc_raise_fmt(lldrmaa_map_get_data(rc), "ll_get_data() return code is non-zero. %s",lldrmaa_err_get_data(rc)); } rc = ll_get_data(step, LL_StepHoldType, &hold_type); if (rc) { fsd_exc_raise_fmt(lldrmaa_map_get_data(rc), "ll_get_data() return code is non-zero. %s",lldrmaa_err_get_data(rc)); } llself->read_job_info(self, step_state, hold_type); fsd_log_info(("Job %s LL State: %d -> DRMAA: %s", self->job_id, step_state, drmaa_job_ps_to_str(self->state))); } } FINALLY { if(job!=NULL) { ll_free_objs(job); ll_deallocate(job); } if(data!=NULL) { ll_free_objs(data); ll_deallocate(data); } if(step!=NULL) { ll_free_objs(step); ll_deallocate(step); } if(host_list!=NULL) fsd_free_vector (host_list); fsd_mutex_unlock( &self->session->drm_connection_mutex ); } END_TRY fsd_log_return(( "" )); } fsd_job_t * lldrmaa_job_new( char *job_id ) { lldrmaa_job_t *self = NULL; self = (lldrmaa_job_t*)fsd_job_new( job_id ); fsd_realloc( self, 1, lldrmaa_job_t ); self->super.control = lldrmaa_job_control; self->super.update_status = lldrmaa_job_update_status; self->missing_time = 0; self->read_job_info_mon = lldrmaa_job_read_job_info_mon; self->read_job_info = lldrmaa_job_read_job_info; return (fsd_job_t*)self; } char * lldrmaa_job_create_req( fsd_drmaa_session_t *session, const fsd_template_t *jt, fsd_environ_t **envp, unsigned bulk, int start, int incr ) { char *volatile cmd_file = NULL; fsd_expand_drmaa_ph_t *volatile expand = NULL; TRY { expand = fsd_expand_drmaa_ph_new( NULL, NULL, fsd_strdup("0") ); cmd_file = (char *) lldrmaa_job_create_command_file( session, jt, envp, expand, bulk, start, incr ); } EXCEPT_DEFAULT { remove(cmd_file); fsd_exc_reraise(); } FINALLY { if( expand ) expand->destroy( expand ); } END_TRY return cmd_file; } static char * internal_map_file( fsd_expand_drmaa_ph_t *expand, const char *path, bool *host_given, const char *name ) { const char *p; for( p = path; *p != ':'; p++ ) if( *p == '\0' ) fsd_exc_raise_fmt( FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_FORMAT, "invalid format of drmaa_%s_path: missing colon", name ); if( host_given ) *host_given = ( p != path ); p++; return expand->expand( expand, fsd_strdup(p), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD | FSD_DRMAA_PH_INCR ); } char * lldrmaa_job_create_command_file( fsd_drmaa_session_t *session, const fsd_template_t *jt, fsd_environ_t **envp, fsd_expand_drmaa_ph_t *expand, unsigned n_jobs, int start, int incr ) { char template_cmd[] = "/tmp/drmaa_cmd_XXXXXX"; int fdt; FILE * fd; if((fdt = mkstemp(template_cmd) ) == -1) { fsd_log_error(("Can't create cmd file")); fsd_exc_raise_msg(FSD_ERRNO_INTERNAL_ERROR,"Can't create cmd file"); } close(fdt); if((fd = fopen(template_cmd,"w") ) != NULL ) { const char *input_path_orig = NULL; const char *output_path_orig = NULL; const char *error_path_orig = NULL; char *volatile input_path = NULL; char *volatile output_path = NULL; char *volatile error_path = NULL; bool input_host = false; bool output_host = false; bool error_host = false; bool join_files = false; const char *value; const char *const *vector; const char *job_category = "default"; fsd_log_debug(("Cmd file: %s opened",template_cmd)); fprintf(fd,"# File created by LoadLeveler DRMAA. If no LL DRMAA applications are running you can delete this file\n"); /* job category */ value = jt->get_attr( jt, DRMAA_JOB_CATEGORY ); if( value ) job_category = value; { char * temp; fsd_conf_option_t *category_value = NULL; category_value = fsd_conf_dict_get( session->job_categories, job_category ); if( category_value != NULL ) { if( category_value->type != FSD_CONF_STRING ) fsd_exc_raise_fmt( FSD_ERRNO_INTERNAL_ERROR, "configuration error: job category should be string" ); temp = fsd_strdup(category_value->val.string); temp = fsd_replace(temp,"@","\n# $"); /* infinite loop when try replace @ with # @ */ temp = fsd_replace(temp,"# $","# @ "); fprintf(fd,"# Job category:%s\n", temp); fsd_log_debug(("Job category:\n%s",temp)); fsd_free(temp); } else { if( value != NULL ) fsd_exc_raise_fmt( FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, "invalid job category: %s", job_category ); } } /* job name */ value = jt->get_attr( jt, DRMAA_JOB_NAME ); if( value ) { fprintf(fd,"# @ job_name = %s\n",value); fsd_log_debug(("# @ job_name = %s",value)); } /* job state at submit */ value = jt->get_attr( jt, DRMAA_JS_STATE ); if( value ) { if( 0 == strcmp( value, DRMAA_SUBMISSION_STATE_ACTIVE ) ) {} else if( 0 == strcmp( value, DRMAA_SUBMISSION_STATE_HOLD ) ) { fprintf(fd,"# @ hold = user\n"); fsd_log_debug(("# @ hold = user")); } else fsd_exc_raise_msg( FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, "invalid value of drmaa_js_state attribute" ); } /* start time */ value = jt->get_attr( jt, DRMAA_START_TIME ); if( value ) { char buf[30]; const time_t time_ll = fsd_datetime_parse( value ); struct tm temp; localtime_r(&time_ll,&temp); strftime(buf, sizeof(buf),"%m/%d/%Y %H:%M:%S", &temp ); fprintf(fd,"# @ startdate = %s\n", buf); fsd_log_debug(("# @ startdate = %s", buf)); fsd_log_debug(( "\n drmaa_start_time: %s -> %ld", value, (long)time_ll )); } /* environment */ vector = jt->get_v_attr( jt, DRMAA_V_ENV ); if( vector ) { const char *const *i; fprintf(fd,"# @ environment ="); fsd_log_debug(("# @ environment =")); for( i = vector; *i; i++ ) { fprintf(fd," %s;",*i); fsd_log_debug((" %s;",*i)); } fprintf(fd," \n"); } /* wall clock time soft and hard limit */ { const char *value2; value = jt->get_attr( jt, DRMAA_WCT_SLIMIT ); value2 = jt->get_attr( jt, DRMAA_WCT_HLIMIT ); if( value && value2) { fprintf(fd,"# @ wall_clock_limit = %s,%s\n",value,value2); fsd_log_debug(("# @ wall_clock_limit = %s,%s",value,value2)); } else if (value) { fprintf(fd,"# @ wall_clock_limit = %s\n",value); fsd_log_debug(("# @ wall_clock_limit = %s",value)); } else if (value2) { fprintf(fd,"# @ wall_clock_limit = ,%s\n",value); fsd_log_debug(("# @ wall_clock_limit = ,%s",value)); } } /* native specification */ value = jt->get_attr( jt, DRMAA_NATIVE_SPECIFICATION ); if( value ) { char * temp = fsd_strdup(value); temp = fsd_replace(temp,"@","\n# $"); /* infinite loop when try replace @ with # @ */ temp = fsd_replace(temp,"# $","# @ "); fprintf(fd,"# Native specification:%s\n", temp); fsd_log_debug(("Native specification:%s", temp)); fsd_free(temp); } { unsigned i; for(i=0;iset(expand, FSD_DRMAA_PH_INCR,fsd_asprintf("%d", start+incr*i)); /* set current value */ /* job working directory */ value = jt->get_attr( jt, DRMAA_WD ); if( value ) { char *cwd_expanded = expand->expand( expand, fsd_strdup(value), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_INCR ); expand->set( expand, FSD_DRMAA_PH_WD, fsd_strdup(cwd_expanded)); fprintf(fd,"# @ initialdir = %s\n",cwd_expanded); fsd_log_debug(("# @ initialdir = %s",cwd_expanded)); fsd_free(cwd_expanded); } TRY { /* input path */ input_path_orig = jt->get_attr( jt, DRMAA_INPUT_PATH ); if( input_path_orig ) { input_path = internal_map_file( expand, input_path_orig, &input_host, "input" ); fsd_log_debug(( "\n drmaa_input_path: %s -> %s", input_path_orig, input_path )); } /* output path */ output_path_orig = jt->get_attr( jt, DRMAA_OUTPUT_PATH ); if( output_path_orig ) { output_path = internal_map_file( expand, output_path_orig, &output_host, "output" ); fsd_log_debug(( "\n drmaa_output_path: %s -> %s", output_path_orig, output_path )); } /* error path */ error_path_orig = jt->get_attr( jt, DRMAA_ERROR_PATH ); if( error_path_orig ) { error_path = internal_map_file( expand, error_path_orig, &error_host, "error" ); fsd_log_debug(( "\n drmaa_error_path: %s -> %s", error_path_orig, error_path )); } /* join files */ value = jt->get_attr( jt, DRMAA_JOIN_FILES ); if( value ) { if( (value[0] == 'y' || value[0] == 'Y') && value[1] == '\0' ) join_files = true; else if( (value[0] == 'n' || value[0] == 'N') && value[1] == '\0' ) join_files = false; else fsd_exc_raise_msg( FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, "invalid value of drmaa_join_files attribute" ); } if( join_files ) { if( output_path == NULL ) fsd_exc_raise_msg( FSD_DRMAA_ERRNO_CONFLICTING_ATTRIBUTE_VALUES, "drmaa_join_files is set and output file is not given" ); if( error_path!=NULL && 0 != strcmp( output_path, error_path ) ) fsd_log_warning(( "Error file was given but will be ignored " "since drmaa_join_files was set." )); if (error_path) fsd_free(error_path); error_path = fsd_strdup(output_path); } else { if( error_path == NULL && output_path ) error_path = fsd_strdup( "/dev/null" ); if( output_path == NULL && error_path ) output_path = fsd_strdup( "/dev/null" ); } /* email addresses to send notifications */ vector = jt->get_v_attr( jt, DRMAA_V_EMAIL ); if( vector && vector[0] ) { /* only to one email address message may be send */ fprintf(fd,"# @ notify_user = %s\n",vector[0]); fsd_log_debug(("# @ notify_user = %s\n",vector[0])); if( vector[1] != NULL ) { fsd_log_error(( "LL only supports one e-mail " "notification address" )); fsd_exc_raise_msg(FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE,"LL only supports one e-mail " "notification address"); } } /* block email */ value = jt->get_attr( jt, DRMAA_BLOCK_EMAIL ); if( value ) { bool block; if( strcmp(value, "0") == 0 ) { block = true; fprintf(fd,"# @ notification = never\n"); fsd_log_debug(("# @ notification = never")); } else if( strcmp(value, "1") == 0 ) block = false; else fsd_exc_raise_msg( FSD_DRMAA_ERRNO_INVALID_ATTRIBUTE_VALUE, "invalid value of drmaa_block_email attribute" ); if( block && output_path == NULL ) { fsd_log_debug(( "output path not set and we want to block e-mail, " "set to /dev/null" )); output_path = fsd_strdup( "/dev/null" ); } } if( input_path ) { fprintf(fd,"# @ input = %s\n", input_path); fsd_log_debug(("# @ input = %s", input_path)); } if( output_path ) { fprintf(fd,"# @ output = %s\n", output_path); fsd_log_debug(("# @ output = %s", output_path)); } if( error_path ) { fprintf(fd,"# @ error = %s\n", error_path); fsd_log_debug(("# @ error = %s", error_path)); } } FINALLY { fsd_free( input_path ); fsd_free( output_path ); fsd_free( error_path ); input_path = NULL; output_path = NULL; error_path = NULL; } END_TRY fprintf(fd,"# @ queue\n"); fsd_log_debug(("# @ queue")); fprintf(fd,"\n"); fprintf(fd,"echo >&2\n"); /* this line forces creation of stderr file */ TRY { const char *command = NULL; char *command_expanded = NULL; const char *const *i; int j; /* remote command */ command = jt->get_attr( jt, DRMAA_REMOTE_COMMAND ); if( command == NULL ) fsd_exc_raise_msg( FSD_DRMAA_ERRNO_CONFLICTING_ATTRIBUTE_VALUES, "drmaa_remote_command not set for job template" ); command_expanded = expand->expand( expand, fsd_strdup(command), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD ); fprintf(fd,"%s", command_expanded); /* we put the commmand at the end of script (instead of using @executable keyword in roder to avoid coping of the binary */ fsd_log_debug(("command = %s\n", command_expanded)); fsd_free(command_expanded); /* arguments list */ vector = jt->get_v_attr( jt, DRMAA_V_ARGV ); if( vector ) { fsd_log_debug(("arguments = ")); for( i = vector, j = 2; *i; i++, j++ ) { char *arg_expanded = expand->expand( expand, fsd_strdup(*i), FSD_DRMAA_PH_HD | FSD_DRMAA_PH_WD ); fprintf(fd," '%s'", arg_expanded); fsd_log_debug(("%s", arg_expanded)); fsd_free(arg_expanded); } } fprintf(fd," \n"); } END_TRY } } fclose(fd); } else { fsd_log_error(("Can't create cmd file")); fsd_exc_raise_msg(FSD_ERRNO_INTERNAL_ERROR,"Can't create cmd file"); } return fsd_strdup(template_cmd); }