Context Navigation

source: trunk/pbs_drmaa/log_reader.c @ 29

Revision 29, 18.6 KB checked in by mmamonski, 13 years ago (diff)
log reder reStructured
Property svn:keywords set to `Id`

Line
1	/* $Id$ */
2	/*
3	* FedStage DRMAA for PBS Pro
4	* Copyright (C) 2006-2007 FedStage Systems
5	*
6	* This program is free software: you can redistribute it and/or modify
7	* it under the terms of the GNU General Public License as published by
8	* the Free Software Foundation, either version 3 of the License, or
9	* (at your option) any later version.
10	*
11	* This program is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	* GNU General Public License for more details.
15	*
16	* You should have received a copy of the GNU General Public License
17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#ifdef HAVE_CONFIG_H
21	# include <config.h>
22	#endif
23
24	#include <stdlib.h>
25	#include <string.h>
26	#include <unistd.h>
27	#include <sys/select.h>
28	#include <sys/stat.h>
29	#include <sys/types.h>
30	#include <dirent.h>
31	#include <fcntl.h>
32
33	#include <pbs_ifl.h>
34	#include <pbs_error.h>
35
36	#include <drmaa_utils/datetime.h>
37	#include <drmaa_utils/drmaa.h>
38	#include <drmaa_utils/iter.h>
39	#include <drmaa_utils/conf.h>
40	#include <drmaa_utils/session.h>
41	#include <drmaa_utils/datetime.h>
42
43	#include <pbs_drmaa/job.h>
44	#include <pbs_drmaa/log_reader.h>
45	#include <pbs_drmaa/session.h>
46	#include <pbs_drmaa/submit.h>
47	#include <pbs_drmaa/util.h>
48	#include <pbs_drmaa/pbs_attrib.h>
49
50	#include <errno.h>
51
52	enum pbsdrmaa_field_id
53	{
54	PBSDRMAA_FLD_ID_DATE = 0,
55	PBSDRMAA_FLD_ID_EVENT = 1,
56	PBSDRMAA_FLD_ID_SRC = 2,
57	PBSDRMAA_FLD_ID_OBJ_TYPE = 3,
58	PBSDRMAA_FLD_ID_OBJ_ID = 4,
59	PBSDRMAA_FLD_ID_MSG = 5
60	};
61
62
63	#define PBSDRMAA_FLD_MSG_0008 "0008"
64	#define PBSDRMAA_FLD_MSG_0010 "0010"
65
66	enum pbsdrmaa_event_type
67	{
68	pbsdrmaa_event_0008 = 8,
69	pbsdrmaa_event_0010 = 10
70	};
71
72	static void pbsdrmaa_read_log();
73
74	static void pbsdrmaa_select_file_wait_thread( pbsdrmaa_log_reader_t * self);
75
76	char pbsdrmaa_read_line_wait_thread( pbsdrmaa_log_reader_t self);
77
78	static time_t pbsdrmaa_parse_log_timestamp(const char timestamp, char unixtime_str, size_t size);
79
80	static char pbsdrmaa_get_exec_host_from_accountig(pbsdrmaa_log_reader_t log_reader, const char *job_id);
81
82	/*
83	* Snippets from log files
84	*
85	* PBS Pro
86	*
87	10/11/2011 14:43:29;0008;Server@nova;Job;2127218.nova;Job Queued at request of mamonski@endor.wcss.wroc.pl, owner = mamonski@endor.wcss.wroc.pl, job name = STDIN, queue = normal
88	10/11/2011 14:43:31;0008;Server@nova;Job;2127218.nova;Job Modified at request of Scheduler@nova.wcss.wroc.pl
89	10/11/2011 14:43:31;0008;Server@nova;Job;2127218.nova;Job Run at request of Scheduler@nova.wcss.wroc.pl on exec_vnode (wn698:ncpus=3:mem=2048000kb)+(wn700:ncpus=3:mem=2048000kb)
90	10/11/2011 14:43:31;0008;Server@nova;Job;2127218.nova;Job Modified at request of Scheduler@nova.wcss.wroc.pl
91	10/11/2011 14:43:32;0010;Server@nova;Job;2127218.nova;Exit_status=0 resources_used.cpupercent=0 resources_used.cput=00:00:00 resources_used.mem=1768kb resources_used.ncpus=6 resources_used.vmem=19228kb resources_used.walltime=00:00:01
92
93	*
94	* Torque
95	*
96	10/11/2011 14:47:59;0008;PBS_Server;Job;15545337.batch.grid.cyf-kr.edu.pl;Job Queued at request of plgmamonski@ui.cyf-kr.edu.pl, owner = plgmamonski@ui.cyf-kr.edu.pl, job name = STDIN, queue = l_short
97	10/11/2011 14:48:23;0008;PBS_Server;Job;15545337.batch.grid.cyf-kr.edu.pl;Job Run at request of root@batch.grid.cyf-kr.edu.pl
98	10/11/2011 14:48:24;0010;PBS_Server;Job;15545337.batch.grid.cyf-kr.edu.pl;Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=720kb resources_used.vmem=13308kb resources_used.walltime=00:00:00
99
100	deleting job:
101	I . PBS Pro
102	a) in Q state
103	10/16/2011 09:49:25;0008;Server@grass1;Job;2178.grass1.man.poznan.pl;Job Queued at request of mmamonski@grass1.man.poznan.pl, owner = mmamonski@grass1.man.poznan.pl, job name = STDIN, queue = workq
104	10/16/2011 09:49:25;0008;Server@grass1;Job;2178.grass1.man.poznan.pl;Job Modified at request of Scheduler@grass1.man.poznan.pl
105	10/16/2011 09:49:37;0008;Server@grass1;Job;2178.grass1.man.poznan.pl;Job to be deleted at request of mmamonski@grass1.man.poznan.pl
106	10/16/2011 09:49:37;0100;Server@grass1;Job;2178.grass1.man.poznan.pl;dequeuing from workq, state 5
107
108
109	b) in R state
110	10/16/2011 09:45:12;0080;Server@grass1;Job;2177.grass1.man.poznan.pl;delete job request received
111	10/16/2011 09:45:12;0008;Server@grass1;Job;2177.grass1.man.poznan.pl;Job sent signal TermJob on delete
112	10/16/2011 09:45:12;0008;Server@grass1;Job;2177.grass1.man.poznan.pl;Job to be deleted at request of mmamonski@grass1.man.poznan.pl
113	10/16/2011 09:45:12;0010;Server@grass1;Job;2177.grass1.man.poznan.pl;Exit_status=271 resources_used.cpupercent=0 resources_used.cput=00:00:00 resources_used.mem=2772kb resources_used.ncpus=1 resources_used.vmem=199288kb resources_used.walltime=00:00:26
114	10/16/2011 09:45:12;0100;Server@grass1;Job;2177.grass1.man.poznan.pl;dequeuing from workq, state 5
115
116	II. Torque
117	a) in Q state
118	10/15/2011 21:19:25;0008;PBS_Server;Job;113045.grass1.man.poznan.pl;Job deleted at request of mmamonski@grass1.man.poznan.pl
119	10/15/2011 21:19:25;0100;PBS_Server;Job;113045.grass1.man.poznan.pl;dequeuing from batch, state EXITING
120
121	b) in R state
122	10/15/2011 21:19:47;0008;PBS_Server;Job;113046.grass1.man.poznan.pl;Job deleted at request of mmamonski@grass1.man.poznan.pl
123	10/15/2011 21:19:47;0008;PBS_Server;Job;113046.grass1.man.poznan.pl;Job sent signal SIGTERM on delete
124	10/15/2011 21:19:47;0010;PBS_Server;Job;113046.grass1.man.poznan.pl;Exit_status=271 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:10
125
126	Log closed:
127	10/16/2011 00:00:17;0002;PBS_Server;Svr;Log;Log closed
128
129	*/
130	pbsdrmaa_log_reader_t *
131	pbsdrmaa_log_reader_new( fsd_drmaa_session_t *session )
132	{
133	pbsdrmaa_log_reader_t *volatile self = NULL;
134
135	fsd_log_enter((""));
136
137	TRY
138	{
139	fsd_malloc(self, pbsdrmaa_log_reader_t );
140
141	self->session = session;
142
143	self->select_file = pbsdrmaa_select_file_wait_thread;
144	self->read_log = pbsdrmaa_read_log;
145
146	self->run_flag = true;
147	self->fhandle = NULL;
148	self->date_changed = true;
149	self->first_open = true;
150
151	}
152	EXCEPT_DEFAULT
153	{
154	if( self != NULL)
155	fsd_free(self);
156
157	fsd_exc_reraise();
158	}
159	END_TRY
160
161	fsd_log_return((""));
162
163	return self;
164	}
165
166
167	void
168	pbsdrmaa_log_reader_destroy ( pbsdrmaa_log_reader_t * self )
169	{
170	fsd_log_enter((""));
171	TRY
172	{
173	if(self != NULL)
174	{
175	fsd_free(self);
176	}
177	}
178	EXCEPT_DEFAULT
179	{
180	fsd_exc_reraise();
181	}
182	END_TRY
183
184	fsd_log_return((""));
185	}
186
187
188	void
189	pbsdrmaa_read_log( pbsdrmaa_log_reader_t * self )
190	{
191	fsd_log_enter((""));
192
193	fsd_mutex_lock( &self->session->mutex );
194
195	TRY
196	{
197	while( self->run_flag )
198	{
199	TRY
200	{
201	char *line = NULL;
202
203	self->select_file(self);
204
205	while ((line = fsd_readline(self->fhandle)) != NULL)
206	{
207	int field_id = PBSDRMAA_FLD_ID_DATE;
208	char *tok_ctx = NULL;
209	char *field_token = NULL;
210	char *event_timestamp = NULL;
211	int event_type = -1;
212	fsd_job_t *job = NULL;
213
214	/* at first detect if this not the end of log file */
215	if (strstr(line, "Log;Log closed")) /TODO try to be more effective and safe /
216	{
217	fsd_log_debug(("WT - Date changed. Closing log file"));
218	self->date_changed = true;
219	}
220
221	for (field_token = strtok_r(line, ";", &tok_ctx); field_token; field_token = strtok_r(NULL, ";", &tok_ctx), field_id++)
222	{
223	if ( field_id == PBSDRMAA_FLD_ID_DATE)
224	{
225	event_timestamp = field_token;
226	}
227	else if ( field_id == PBSDRMAA_FLD_ID_EVENT)
228	{
229	if (strncmp(field_token, PBSDRMAA_FLD_MSG_0008, 4) == 0)
230	event_type = pbsdrmaa_event_0008;
231	else if (strncmp(field_token, PBSDRMAA_FLD_MSG_0010, 4) == 0)
232	event_type = pbsdrmaa_event_0010;
233	else
234	break; /we are interested only in the above log messages /
235	}
236	else if ( field_id == PBSDRMAA_FLD_ID_SRC)
237	{
238	/* not used ignore */
239	}
240	else if (field_id == PBSDRMAA_FLD_ID_OBJ_TYPE)
241	{
242	if (strncmp(field_token, "Job", 3) != 0)
243	break; /* we are interested only in job events */
244	}
245	else if (field_id == PBSDRMAA_FLD_ID_OBJ_ID)
246	{
247	const char *event_jobid = field_token;
248
249	TRY
250	{
251	job = self->session->get_job( self->session, event_jobid );
252
253	if( job )
254	{
255	fsd_log_debug(("WT - Found job event: %s", event_jobid));
256	}
257	else
258	{
259	fsd_log_debug(("WT - Unknown job: %s", event_jobid)); /* Not a DRMAA job */
260	break;
261	}
262	}
263	END_TRY
264	}
265	else if (field_id == PBSDRMAA_FLD_ID_MSG)
266	{
267	char *msg = field_token;
268	struct batch_status status;
269	struct attrl *attribs = NULL;
270	bool in_running_state = false;
271
272	if (event_type == pbsdrmaa_event_0008 && strncmp(msg, "Job Queued", 10) == 0)
273	{
274	/* Queued
275	* PBS Pro: 10/11/2011 14:43:29;0008;Server@nova;Job;2127218.nova;Job Queued at request of mamonski@endor.wcss.wroc.pl, owner = mamonski@endor.wcss.wroc.pl, job name = STDIN, queue = normal
276	* Torque: 10/11/2011 14:47:59;0008;PBS_Server;Job;15545337.batch.grid.cyf-kr.edu.pl;Job Queued at request of plgmamonski@ui.cyf-kr.edu.pl, owner = plgmamonski@ui.cyf-kr.edu.pl, job name = STDIN, queue = l_short
277	*/
278	char *p_queue = NULL;
279
280	if ((p_queue = strstr(msg,"queue =")) == NULL)
281	fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"No queue attribute found in log line = %s", line);
282
283	/* != Job deleted and Job to be deleted*/
284	#ifdef PBS_PROFESSIONAL
285	if (field[4] != 't' && field[10] != 'd')
286	{
287	#else
288	if (field[4] != 'd')
289	{
290	#endif
291	struct_state.value = fsd_asprintf("%c",field[n]);
292	(void)pbsdrmaa_parse_log_timestamp(event_timestamp, timestamp_unix, sizeof(timestamp_unix));
293
294	in_running_state = true;
295
296	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_JOB_STATE, "R");
297	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_START_TIME, timestamp_unix);
298	#ifdef PBS_PROFESSIONAL
299	{
300	char *p_vnode = NULL;
301	if ((p_vnode = strstr(field, "exec_vnode")))
302	{
303	last_attr->next = &struct_exec_vnode;
304	last_attr = &struct_exec_vnode;
305	struct_exec_vnode.name = "exec_vnode";
306	struct_exec_vnode.next = NULL;
307	struct_exec_vnode.value = fsd_strdup(p_vnode + 11);
308	}
309	}
310	#endif
311	}
312	#ifndef PBS_PBS_PROFESSIONAL
313	else if (event_type == pbsdrmaa_event_0008 && strncmp(msg, "Job deleted", 11))
314	#else
315	else if (event_type == pbsdrmaa_event_0008 && strncmp(msg, "Job to be deleted", 17))
316	#endif
317	{
318	/* Deleted
319	* PBS Pro: 10/16/2011 09:45:12;0008;Server@grass1;Job;2177.grass1.man.poznan.pl;Job to be deleted at request of mmamonski@grass1.man.poznan.pl
320	* Torque: 10/15/2011 21:19:25;0008;PBS_Server;Job;113045.grass1.man.poznan.pl;Job deleted at request of mmamonski@grass1.man.poznan.pl
321	*/
322	char timestamp_unix[64];
323
324	(void)pbsdrmaa_parse_log_timestamp(event_timestamp, timestamp_unix, sizeof(timestamp_unix));
325
326	if (job->state < DRMAA_PS_RUNNING)
327	{
328	fsd_log_info(("Job %s killed before entering running state (%d).", job->job_id, job->state));
329	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_JOB_STATE, "C");
330	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_MTIME, timestamp_unix);
331	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_EXIT_STATUS, "-2");
332	}
333	else
334	{
335	break; /* job was started, ignore, wait for Exit_status message */
336	}
337	}
338	else if (event_type == pbsdrmaa_event_0010 && strncmp(msg, "Exit_status=", 12))
339	{
340	/* Completed:
341	* PBS Pro: 10/11/2011 14:43:32;0010;Server@nova;Job;2127218.nova;Exit_status=0 resources_used.cpupercent=0 resources_used.cput=00:00:00 resources_used.mem=1768kb resources_used.ncpus=6 resources_used.vmem=19228kb resources_used.walltime=00:00:01
342	* Torque: 10/11/2011 14:48:24;0010;PBS_Server;Job;15545337.batch.grid.cyf-kr.edu.pl;Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=720kb resources_used.vmem=13308kb resources_used.walltime=00:00:00
343	*/
344	char timestamp_unix[64];
345	time_t timestamp_time_t = pbsdrmaa_parse_log_timestamp(event_timestamp, timestamp_unix, sizeof(timestamp_unix));
346	char *tok_ctx2 = NULL;
347	char *token = NULL;
348
349	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_JOB_STATE, "C");
350	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_MTIME, timestamp_unix);
351
352	/* tokenize !!! */
353	for (token = strtok_r(msg, " ", &tok_ctx2); token; token = strtok_r(NULL, " ", &tok_ctx2))
354	{
355	if (strncmp(token, "Exit_status=", 12) == 0)
356	{
357	token[12] = '\0';
358	attribs = pbsdrmaa_add_attr(attribs, token, token + 12);
359	fsd_log_info(("WT - Completion of job %s (Exit_status=%s) detected after %d seconds", job->job_id, token+12, (int)(time(NULL) - timestamp_time_t) ));
360	}
361	else if (strncmp(token, "resources_used.cput=", 20) == 0)
362	{
363	token[20] = '\0';
364	attribs = pbsdrmaa_add_attr(attribs, token, token + 20);
365	}
366	else if (strncmp(token, "resources_used.mem=", 19) == 0)
367	{
368	token[19] = '\0';
369	attribs = pbsdrmaa_add_attr(attribs, token, token + 19);
370	}
371	else if (strncmp(token, "resources_used.vmem=", 20) == 0)
372	{
373	token[20] = '\0';
374	attribs = pbsdrmaa_add_attr(attribs, token, token + 20);
375	}
376	else if (strncmp(token, "resources_used.walltime=", 24) == 0)
377	{
378	token[24] = '\0';
379	attribs = pbsdrmaa_add_attr(attribs, token, token + 24);
380	}
381	}
382
383	if (!job->execution_hosts)
384	{
385	char *exec_host = NULL;
386	fsd_log_info(("WT - No execution host information for job %s. Reading accounting logs...", job->job_id));
387	exec_host = pbsdrmaa_get_exec_host_from_accountig(self, job->job_id);
388	attribs = pbsdrmaa_add_attr(attribs, PBSDRMAA_EXECUTION_HOST, exec_host);
389	fsd_free(exec_host);
390	}
391	}
392	else
393	{
394	break; /* ignore other job events*/
395	}
396
397	if ( in_running_state )
398	{
399	fsd_log_debug(("WT - forcing update of job: %s", job->job_id ));
400	TRY
401	{
402	job->update_status( job );
403	}
404	EXCEPT_DEFAULT
405	{
406	/TODO: distinguish between invalid job and internal errors /
407	fsd_log_debug(("Job finished just after entering running state: %s", job->job_id));
408	}
409	END_TRY
410	}
411	else
412	{
413	fsd_log_debug(("WT - updating job: %s", job->job_id ));
414	status.name = job->job_id;
415	status.attribs = attribs;
416
417	((pbsdrmaa_job_t *)job)->update( job, &status );
418
419	pbsdrmaa_free_attrl(attribs); /* TODO free on exception */
420	}
421
422	fsd_cond_broadcast( &job->status_cond);
423	fsd_cond_broadcast( &self->session->wait_condition );
424
425	if ( job )
426	job->release( job );
427
428	fsd_free(line); /* TODO free on exception */
429	}
430	else
431	{
432	fsd_assert(0); /not reached /
433	}
434	}
435
436	} /* end of while getline loop */
437
438	{ /* poll on log file */
439	struct timeval timeout_tv;
440	fd_set log_fds;
441
442	fsd_mutex_unlock( &self->session->mutex );
443
444	FD_ZERO(&log_fds);
445	FD_SET(fileno(self->fhandle), &log_fds);
446
447	timeout_tv.tv_sec = 1;
448	timeout_tv.tv_usec = 0;
449
450	/* ignore return value - the next get line call will handle IO errors */
451	(void)select(1, &log_fds, NULL, NULL, &timeout_tv);
452
453	fsd_mutex_lock( &self->session->mutex );
454
455	self->run_flag = self->session->wait_thread_run_flag;
456	}
457	}
458	EXCEPT_DEFAULT
459	{
460	const fsd_exc_t *e = fsd_exc_get();
461	/* Its better to exit and communicate error rather then let the application to hang */
462	fsd_log_fatal(( "Exception in wait thread: <%d:%s>. Exiting !!!", e->code(e), e->message(e) ));
463	exit(1);
464	}
465	END_TRY
466	}
467
468	if(self->fhandle)
469	fclose(self->fhandle);
470
471	fsd_log_debug(("WT - Log file closed"));
472	}
473	FINALLY
474	{
475	fsd_log_debug(("WT - Terminated."));
476	fsd_mutex_unlock( &self->session->mutex ); /**/
477	}
478	END_TRY
479
480	fsd_log_return((""));
481	}
482
483	void
484	pbsdrmaa_select_file_wait_thread ( pbsdrmaa_log_reader_t * self )
485	{
486	pbsdrmaa_session_t pbssession = (pbsdrmaa_session_t) self->session;
487
488	if (self->date_changed)
489	{
490	char * log_path = NULL;
491	int num_tries = 0;
492	struct tm tm;
493
494	fsd_log_enter((""));
495
496	if(!self->first_open)
497	time(&self->t);
498	else
499	self->t = pbssession->log_file_initial_time;
500
501	localtime_r(&self->t,&tm);
502
503	#define DRMAA_WAIT_THREAD_MAX_TRIES (12)
504	/* generate new date, close file and open new */
505	log_path = fsd_asprintf("%s/server_logs/%04d%02d%02d", pbssession->pbs_home, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);
506
507	if(self->fhandle)
508	fclose(self->fhandle);
509
510	fsd_log_info(("Opening log file: %s",log_path));
511
512	retry:
513	if ((self->fhandle = fopen(log_path,"")) == NULL && (num_tries > DRMAA_WAIT_THREAD_MAX_TRIES \|\| self->first_open))
514	{
515	fsd_log_error(("Can't open log file. Verify pbs_home. Running standard wait_thread."));
516	fsd_log_error(("Remember that without keep_completed set the standard wait_thread won't provide information about job exit status"));
517	/pbssession->super.enable_wait_thread = false;/ /* run not wait_thread */
518	pbssession->wait_thread_log = false;
519	pbssession->super.wait_thread = pbssession->super_wait_thread;
520	pbssession->super.wait_thread(self->session);
521	}
522	else if ( self->fhandle == NULL )
523	{ /* Torque seems not to create a new file immediately after the old one is closed */
524	fsd_log_warning(("Can't open log file: %s. Retries count: %d", log_path, num_tries));
525	num_tries++;
526	sleep(2 * num_tries);
527	goto retry;
528	}
529
530	fsd_free(log_path);
531
532	fsd_log_debug(("Log file opened"));
533
534	if(self->first_open)
535	{
536	fsd_log_debug(("Log file lseek"));
537
538	if(fseek(self->fhandle, pbssession->log_file_initial_size, SEEK_SET) == (off_t) -1)
539	{
540	fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"fseek error");
541	}
542	self->first_open = false;
543	}
544
545	self->date_changed = false;
546
547	fsd_log_return((""));
548	}
549	}
550
551	time_t
552	pbsdrmaa_parse_log_timestamp(const char timestamp, char unixtime_str, size_t size)
553	{
554	struct tm temp_time_tm;
555	memset(&temp_time_tm, 0, sizeof(temp_time_tm));
556	temp_time_tm.tm_isdst = -1;
557
558	if (strptime(timestamp, "%m/%d/%Y %H:%M:%S", &temp_time_tm) == NULL)
559	{
560	fsd_exc_raise_fmt(FSD_ERRNO_INTERNAL_ERROR,"WT - failed to parse log timestamp: %s", timestamp);
561	}
562	else
563	{
564	time_t temp_time = mktime(&temp_time_tm);
565	snprintf(unixtime_str, size, "%lu", temp_time);
566	return temp_time;
567	}
568	}
569
570	char *
571	pbsdrmaa_get_exec_host_from_accountig(pbsdrmaa_log_reader_t * log_reader, const char *job_id)
572	{
573	/* TODO: implement */
574	return NULL;
575	}
576
577

Note: See TracBrowser for help on using the repository browser.

Download in other formats: