#!/usr/bin/perl ############################################################################### # # sjstat - List attributes of jobs under SLURM control # ############################################################################### # Copyright (C) 2007 The Regents of the University of California. # Copyright (C) 2008-2009 Lawrence Livermore National Security. # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). # Written by Phil Eckert . # CODE-OCEC-09-009. All rights reserved. # # This file is part of Slurm, a resource management program. # For details, see . # Please also read the included file: DISCLAIMER. # # SLURM is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free # Software Foundation; either version 2 of the License, or (at your option) # any later version. # # In addition, as a special exception, the copyright holders give permission # to link the code of portions of this program with the OpenSSL library under # certain conditions as described in each individual source file, and # distribute linked combinations including the two. You must obey the GNU # General Public License in all respects for all of the code used other than # OpenSSL. If you modify file(s) with this exception, you may extend this # exception to your version of the file(s), but you are not obligated to do # so. If you do not wish to do so, delete this exception statement from your # version. If you delete this exception statement from all source files in # the program, then also delete it here. # # SLURM is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along # with SLURM; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Based off code with permission copyright 2006, 2007 Cluster Resources, Inc. ############################################################################### # # Man page stuff. # BEGIN { # Just dump the man page in *roff format and exit if --roff specified. foreach my $arg (@ARGV) { if ($arg eq "--") { last; } elsif ($arg eq "--roff") { use Pod::Man; my $parser = Pod::Man->new (section => 1); $parser->parse_from_file($0, \*STDOUT); exit 0; } } } use strict; use Getopt::Long 2.24 qw(:config no_ignore_case); use autouse 'Pod::Usage' => qw(pod2usage); # # Global Variables. # my ($help, $man, $pool, $running, $verbose); my (%MaxNodes, %MaxTime); # # Check SLURM status. # isslurmup(); # # Get user options. # get_options(); # # Get partition information from scontrol, used # currently in conjunction with the sinfo data.. # do_scontrol_part(); # # Get and display the sinfo data. # do_sinfo(); # # If the -c option was entered, stop here. # exit if ($pool); # # Get and display the squeue data. # do_squeue(); exit; # # Get the SLURM partitions information. # sub do_sinfo { my (@s_part, @s_mem, @s_cpu, @s_feat, @s_active, @s_idle, @s_out, @s_total, @s_usable); # # Get the partition and node info. # my $options = "\"%9P %7m %.4c %.22F %f\""; my $ct = 0; my @sin = `sinfo -e -o $options`; foreach my $tmp (@sin) { next if ($tmp =~ /^PARTITION/); chomp $tmp; my @line = split(' ',$tmp); $s_part[$ct] = $line[0]; $s_mem[$ct] = $line[1]; $s_cpu[$ct] = $line[2]; # # Split the status into various components. # my @fields = split(/\//, $line[3]); $s_active[$ct] = $fields[0]; $s_idle[$ct] = $fields[1]; $s_out[$ct] = $fields[2]; $s_total[$ct] = $fields[3]; $s_usable[$ct] = $s_total[$ct] - $s_out[$ct]; $s_feat[$ct] = ($line[4] .= " "); $s_feat[$ct] =~ s/\(null\)//g; $ct++; } printf("\nScheduling pool data:\n"); if ($verbose) { printf("----------------------------------------------------------------------------------\n"); printf(" Total Usable Free Node Time Other \n"); printf("Pool Memory Cpus Nodes Nodes Nodes Limit Limit traits \n"); printf("----------------------------------------------------------------------------------\n"); } else { printf("-------------------------------------------------------------\n"); printf("Pool Memory Cpus Total Usable Free Other Traits \n"); printf("-------------------------------------------------------------\n"); } for (my $i = 0; $i < $ct; $i++) { if ($verbose) { my $p = $s_part[$i]; $p =~ s/\*//; printf("%-9s %7dMb %5s %6s %7s %6s %6s %10s %-s\n", $s_part[$i], $s_mem[$i], $s_cpu[$i], $s_total[$i], $s_usable[$i], $s_idle[$i], $MaxNodes{$p}, $MaxTime{$p}, $s_feat[$i]); } else { printf("%-9s %7dMb %5s %6s %6s %6s %-s\n", $s_part[$i], $s_mem[$i], $s_cpu[$i], $s_total[$i], $s_usable[$i], $s_idle[$i], $s_feat[$i]); } } printf("\n"); return; } # # Get the SLURM queues. # sub do_squeue { my (@s_job, @s_user, @s_nodes, @s_status, @s_begin, @s_limit, @s_start, @s_pool, @s_used, @s_master); # # Base options on whether this partition is node or process scheduled. # my ($type, $options); my $rval = system("scontrol show config | grep cons_res >> /dev/null"); if ($rval) { $type = "Nodes"; $options = "\"%8i %8u %.6D %2t %S %.12l %.9P %.11M %1000R\""; } else { $type = "Procs"; $options = "\"%8i %8u %.6C %2t %S %.12l %.9P %.11M %1000R\""; } # # Get the job information. # my $ct = 0; my $pat = "tr -s '[' '\000' |cut -d'-' -f 1 | cut -d',' -f 1"; my @sout = `squeue -o $options`; foreach my $tmp (@sout) { next if ($tmp =~ /^JOBID/); next if ($running && $tmp =~ / PD /); chomp $tmp; my @line = split(' ', $tmp); $s_job[$ct] = $line[0]; $s_user[$ct] = $line[1]; $s_nodes[$ct] = $line[2]; $s_status[$ct] = $line[3]; $line[4] =~ s/^.....//; $line[4] = "N/A" if ($line[3] =~ /PD/); $s_begin[$ct] = $line[4]; $s_limit[$ct] = $line[5]; if ($line[5] eq "UNLIMITED") { $s_limit[$ct] = $line[5]; } else { $s_limit[$ct] = convert_time($line[5]); } $s_pool[$ct] = $line[6]; $s_used[$ct] = $line[7]; # # Only keep the master node from the nodes list. # $line[8] =~ s/\[([0-9.]*).*/$1/; $s_master[$ct] = $line[8]; $ct++; } printf("Running job data:\n"); if ($verbose) { printf("---------------------------------------------------------------------------------------------------\n"); printf(" Time Time Time \n"); printf("JobID User $type Pool Status Used Limit Started Master/Other \n"); printf("---------------------------------------------------------------------------------------------------\n"); } else { printf("----------------------------------------------------------------------\n"); printf("JobID User $type Pool Status Used Master/Other \n"); printf("----------------------------------------------------------------------\n"); } for (my $i = 0; $i < $ct; $i++) { if ($verbose) { printf("%-8s %-8s %6s %-9s %-7s %10s %11s %14s %.12s\n", $s_job[$i], $s_user[$i], $s_nodes[$i], $s_pool[$i], $s_status[$i], $s_used[$i], $s_limit[$i], $s_begin[$i], $s_master[$i]); } else { printf("%-8s %-8s %6s %-9s %-7s %10s %.12s\n", $s_job[$i], $s_user[$i], $s_nodes[$i], $s_pool[$i], $s_status[$i], $s_used[$i], $s_master[$i]); } } printf("\n"); return; } # # Get the SLURM partitions. # sub do_scontrol_part { # # Get All partition data Don't need it all now, but # it may be useful later. # my @scon = `scontrol show part`; my $part; foreach my $tmp (@scon) { chomp $tmp; my @line = split(' ',$tmp); ($part) = ($tmp =~ m/PartitionName=(\S+)/) if ($tmp =~ /PartitionName=/); ($MaxTime{$part}) = ($tmp =~ m/MaxTime=(\S+)\s+/) if ($tmp =~ /MaxTime=/); ($MaxNodes{$part}) = ($tmp =~ m/MaxNodes=(\S+)\s+/) if ($tmp =~ /MaxNodes=/); $MaxTime{$part} =~ s/UNLIMITED/UNLIM/ if ($MaxTime{$part}); $MaxNodes{$part} =~ s/UNLIMITED/UNLIM/ if ($MaxNodes{$part}); } return; } # # Show the man page. # sub show_man { if ($< == 0) { # Cannot invoke perldoc as root my $id = eval { getpwnam("nobody") }; $id = eval { getpwnam("nouser") } unless defined $id; $id = -2 unless defined $id; $< = $id; printf("\n You can not do this as root!\n\n"); exit 1; } $> = $<; # Disengage setuid $ENV{PATH} = "/bin:/usr/bin"; # Untaint PATH delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; if ($0 =~ /^([-\/\w\.]+)$/) { $0 = $1; } # Untaint $0 else { die "Illegal characters were found in \$0 ($0)\n"; } pod2usage(-exitstatus => 0, -verbose => 2); return; } # # Convert the time to a better format. # sub convert_time { my $val = shift(@_); my $tmp; my @field = split(/-|:/, $val); if (@field == 4) { $tmp = ($field[0]*24)+$field[1] . ':'.$field[2] . ':' . $field[3]; } else { $tmp = sprintf("%8s",$val); } return($tmp); } # # Get options. # sub get_options { GetOptions( 'help|h|?' => \$help, 'man' => \$man, 'v' => \$verbose, 'r' => \$running, 'c' => \$pool, ) or usage(1); show_man() if ($man); usage(0) if ($help); return; } # # Usage. # sub usage { my $eval = shift(@_); # # Print usage instructions and exit. # print STDERR "\nUsage: sjstat [-h] [-c] p\[-man] [-r] [-v]\n"; printf("\ -h shows usage. -c shows computing resources info only. -man shows man page. -r show only running jobs. -v is for the verbose mode.\n Output is very similar to that of squeue. \n\n"); exit($eval); } # # Determine if SLURM is available. # sub isslurmup { my $out = `scontrol show part 2>&1`; if ($?) { printf("\n SLURM is not communicating.\n\n"); exit(1); } return; } __END__ =head1 NAME B - List attributes of jobs under the SLURM control =head1 SYNOPSIS B [B<-h> ] [B<-c>] [B<-r> ] [B<-v>] =head1 DESCRIPTION The B command is used to display statistics of jobs under control of SLURM. The output is designed to give information on the resource usage and availablilty, as well as information about jobs that are currently active on the machine. This output is built using the SLURM utilities, sinfo, squeue and scontrol, the man pages for these utilites will provide more information and greater depth of understanding. =head1 OPTIONS =over 4 =item B<-h> Display a brief help message =item B<-c> Display the computing resource information only. =item B<-man> Show the man page. =item B<-r> Display only the running jobs. =item B<-v> Display more verbose information. =back =head1 EXAMPLE The following is a basic request for status. > sjstat Scheduling pool data: ------------------------------------------------------------ Pool Memory Cpus Total Usable Free Other Traits ------------------------------------------------------------ pdebug 15000Mb 8 32 32 24 (null) pbatch* 15000Mb 8 1072 1070 174 (null) Running job data: ------------------------------------------------------------------- JobID User Nodes Pool Status Used Master/Other ------------------------------------------------------------------- 395 mary 1000 pbatch PD 0:00 (JobHeld) 396 mary 1000 pbatch PD 0:00 (JobHeld) 375 sam 1000 pbatch CG 0:00 (JobHeld) 388 fred 32 pbatch R 25:27 atlas89 361 harry 512 pbatch R 1:01:12 atlas618 1077742 sally 8 pdebug R 20:16 atlas18 The Scheduling data contains information pertaining to the: Pool a set of nodes Memory the amount of memory on each node Cpus the number of cpus on each node Total the total number of nodes in the pool Usable total usaable nodes in the pool Free total nodes that are currently free The Running job data contains information pertaining to the: JobID the SLURM job id User owner of the job Nodes nodes required, or in use by the job (Note: On cpu scheduled machines, this field will be labled "Procs" show the number of processors the job is using.) Pool the Pool required or in use by the job Status current status of the job Used Wallclick time used by the job Master/Other Either the Master (head) node used by the job, or may indicate furhter status of a pending, or completing job. The common status values are: R The job is running PD The job is Pending CG The job is Completing These are states reproted by SLURM and more elaborate docuemntation can be found in the squeue/sinfo man pages. An example of the -v option. Scheduling pool data: ----------------------------------------------------------------------------- Total Usable Free Node Time Other Pool Memory Cpus Nodes Nodes Nodes Limit Limit Traits ----------------------------------------------------------------------------- pdebug 15000Mb 8 32 32 24 16 30 (null) pbatch* 15000Mb 8 1072 1070 174 UNLIM UNLIM (null) Running job data: --------------------------------------------------------------------------------------------------- Time Time Time JobID User Nodes Pool Status Used Limit Started Master/Other --------------------------------------------------------------------------------------------------- 38562 tom 4 pbatch PD 0:00 1:00:00 01-14T18:11:22 (JobHeld) The added fields to the "Scheduling pool data" are: Node Limit SLURM imposed node limit. Time Limit SLURM imposed time limit, value in minutes. The added fields to the "Running job data" are: Limit Time limit of job. Start Start time of job. =head1 REPORTING BUGS Report bugs to =cut