--27ZtN5FSuKKSZcBU
Content-Type: multipart/mixed; boundary="zjcmjzIkjQU2rmur"
Content-Disposition: inline
--zjcmjzIkjQU2rmur
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable
* Falk, Martin IZ/HZA-ICS <falkmrt AT de.ina DOT com> [2003-12-17 16:32]:
> Hello all,
> =20
> for the command
> =20
> bpdbjobs -report
> [-M <master_server>] [-format_file <pathname>] [-file <pathname>]
> [-header] [-append] [-verbose]
-format_file is no longer an option in 4.5 it was replaced by options in
the bp.conf (which I hate, by the way). Another option you may want to
try is a Python 2.3 script I wrote to parse the -all_columns output.
It's not too long, so I will attach it here.
--=20
David Rock
david AT graniteweb DOT com
--zjcmjzIkjQU2rmur
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="bpdbreport.py"
Content-Transfer-Encoding: quoted-printable
#!/usr/local/bin/python
#
# bpdbreport.py
#
# Take input from bpdbjobs -report -all_columns=20
# and change info into human readable format
#
# Changelog:
#
# 2003-12-03: DWR
# Changed end time commandline entry to include end date
# It was taking the date as dd/mmm/yyyy and assuming 00:00:00
# for the time, which does NOT include the data from that day.
# 2003-10-30: DWR
# Discovered what job type 5 is: Import
# Discovered what job type 3 is: Verify
# Added import and verify categories
# 2003-10-28: DWR
# Added --mdy switch to readability output so that the interim
# reporting solution can continue to function. The default format
# for dates is dd/mmm/yyyy, and this allows for usage of dd/mm/yyyy
# 2003-10-21: DWR
# Added readability data handling to individual try information
# Changed line processing to use the csv module in python 2.3
# instead of the messy logic to walk the line looking for special
# cases that broke because of embedded and escaped commas. Because
# of this, the script now REQUIRES python 2.3
# 2003-10-02: DWR
# Started adding Job types for Netbackup 4.5, still need types
# 3 and 5
# 2003-09-17: DWR
# Cleaned up logic to find escaped commas in text so that the=20
# fields will parse correctly. Known sections where this applies
# are; path_last_written, filelist, and trystatusdetails.=20
# 2003-09-16: DWR
# Added more debug code for -d switch. This is almost completely
# for my benefit only to aid in fixing other problems with the
# script itself. Users would generally not be using it from day
# to day. Problem lines will generate ERROR: line messages to
# stderr nw instead of generating a traceback. Most of this exception
# handling is wrapped around the process_line module since that is
# where the bulk of the problems would occur.
# 2003-08-11: DWR
# Added readability module to the all_data output so human
# readable output is now possible (-a -v).
# 2003-08-08: DWR
# Fixed bug where embedded commas in path last written field
# would screw up field splitting. Also started to add debug=20
# mode switch to get more info that can be collected on stderr
import os
import re
import csv
import sys
import time
import types
import getopt
import string
import fileinput
# definitions for indexed job columns
job_type =3D { '0' : 'Backup',
'1' : 'Archive',
'2' : 'Restore',
'3' : 'Verify',
'4' : 'Duplicate',
'5' : 'Import',
'6' : 'DB Backup',
'7' : 'Vault'
}
job_state =3D { '0' : 'Queued', '1' : 'Active', '2' : 'Re-Queued', '3' : 'D=
one' }
sched_type =3D { '0': 'Full', '1' : 'Differential', '2' : 'User Backup', '3=
' : 'User Archive', '4' : 'Cumulative' }
def usage():
print >>sys.stderr, '''\nbpdbreport.py usage:
bpdbreport.py [switches] <filelist> | -
-a all data format (includes try information)
-d run in debug mode (outputs to stderr)
-f format_file column output format file
-s dd/mmm/yyyy define start time (default is epoch)
-e dd/mmm/yyyy define end time (default is current localtime)
-h print this help and exit
--mdy change verbose date output format to mm/dd/yyyy
--usage print detailed help message and exit
-v verbose (human readable output)
Default output is the first 32 columns from bpdbjobs -report -all_colum=
ns
formatted data. Columns can be defined by format file (see --usage for=
=20
sample format file)
examples:
get all entries in verbose format from stdin:
bpdbreport.py -s 05/may/2003 -e 05/jun/2003 -v -
get data from file named all_columns.output and display columns def=
ined
in sample.fmt file
bpdbreport.py -f sample.fmt all_columns.output
Lines that generate bad data (mostly because of bugs or bad commas) will
spit out error lines to stderr in the format of:
ERROR: inputline
'''
def detailed_usage():
usage()
print >>sys.stderr, '''
# sample format file for column output
# This will skip all lines that start with # and
# all whitespace lines.
# Any lines that are incorrect will be dropped
jobid
jobtype
state
status
class
sched
client
server
start
elapsed
end
stunit
try
operation
kbytes
files
path_last_written
percent
jobpid
owner
subtype
classtype
schedtype
priority
group
master_server
retention_units
retention_period
compression
kbyteslastwritten
fileslastwritten
filelistcount'''
def sec_to_hms( input ):
input =3D seconds =3D int(input)
hours =3D seconds / 3600
seconds =3D seconds - hours*3600
minutes =3D seconds / 60
seconds =3D seconds - minutes*60
return (hours,minutes,seconds)
def process_line(buffer):
dict =3D {}
idx =3D 0
info_labels =3D ( 'jobid', 'jobtype', 'state', 'status', 'class', 'sche=
d',=20
'client', 'server', 'start', 'elapsed', 'end', 'stunit'=
,=20
'try', 'operation', 'kbytes', 'files', 'path_last_writt=
en',#17
'percent', 'jobpid', 'owner', 'subtype', 'classtype',=
=20
'schedtype', 'priority', 'group', 'master_server',=20
'retention_units', 'retention_period', 'compression',=
=20
'kbyteslastwritten', 'fileslastwritten', 'filelistcount=
' )
try_labels1 =3D ( 'trypid', 'trystunit', 'tryserver', 'trystarted', 'tr=
yelapsed',=20
'tryended', 'trystatus', 'trystatusdescription', 'tryst=
atuscount' )
try_labels2 =3D ( 'trybyteswritten','tryfileswritten' )
for label in info_labels:
dict[label] =3D buffer[idx]
idx +=3D 1
try:
if dict['filelistcount'] > 0:
for f in range ( idx, idx+int(dict['filelistcount']) ):
try:
dict['filelist'].append(buffer[idx])
except:
dict['filelist'] =3D [buffer[idx]]
idx +=3D 1
dict['trycount'] =3D buffer[idx]
idx +=3D 1
for job_try in range(1,int(dict['trycount'])+1):
try_idx =3D 'try'+str(job_try)
dict[try_idx] =3D {}
for trylabel in try_labels1:
dict[try_idx][trylabel] =3D buffer[idx]
idx +=3D 1
if dict[try_idx]['trystatuscount'] > 0:
for f in range ( idx, idx+int(dict[try_idx]['trystatuscount=
']) ):
try:
dict[try_idx]['trystatuslines'].append(buffer[idx])
except:
dict[try_idx]['trystatuslines'] =3D [buffer[idx]]
idx +=3D 1
for trylabel in try_labels2:
dict[try_idx][trylabel] =3D buffer[idx]
idx +=3D 1
return dict, 0, False
except:
return dict, sys.exc_info(),buffer
def get_output_cols( format_file ):
try:
col_fp =3D open( format_file, 'r' )
except:
print >>sys.stderr, 'could not open format file'
return None
for line in col_fp:
line =3D line.rstrip('\n')
buf =3D line.split('#',1)
if buf[0]:
try:
col_fmt.append(buf[0].strip())
except:
col_fmt =3D [buf[0].strip()]
col_fp.close()
return col_fmt
def print_dict(d,k,t):
if debug_mode:
print >>sys.stderr,'DEBUG: ',k,'{'
keys =3D d.keys()
keys.sort()
for item in keys:
if type(d[item]) is types.DictType:
print_dict(d[item],item,t+1)
elif type(d[item]) is types.ListType:
print_list(d[item],item,t+1)
else:
print >>sys.stderr,'DEBUG: ','\t'*t+item,':',d[item]
print >>sys.stderr,'DEBUG: ','}'
else:
print k,'{'
keys =3D d.keys()
keys.sort()
for item in keys:
if type(d[item]) is types.DictType:
print_dict(d[item],item,t+1)
elif type(d[item]) is types.ListType:
print_list(d[item],item,t+1)
else:
if verbose:
print '\t'*t+item,':',readability( item, d[item] )
else:
print '\t'*t+item,':',d[item]
print '}'
def print_list(l,k,t):
if debug_mode:
idx =3D 0
print >>sys.stderr,'DEBUG: ','\t'*(t-1)+k,'{'
for item in l:
if type(item) is types.DictType:
print_dict(l[idx],item,t+1)
elif type(item) is types.ListType:
print_list(l[idx],item,t+1)
else:
print >>sys.stderr,'DEBUG: ','\t'*t+l[idx]
idx +=3D 1
print >>sys.stderr,'DEBUG: ','\t'*(t-1)+'}'
else:
idx =3D 0
print '\t'*(t-1)+k,'{'
for item in l:
if type(item) is types.DictType:
print_dict(l[idx],item,t+1)
elif type(item) is types.ListType:
print_list(l[idx],item,t+1)
else:
if verbose:
print '\t'*t+readability( item, l[idx] )
else:
print '\t'*t+l[idx]
idx +=3D 1
print '\t'*(t-1)+'}'
def output_data( d,col_fmt ):
keys =3D d.keys()
keys.sort()
if all_data:
for key in keys:
print key,'{'
k =3D d[key].keys()
k.sort()
for item in k:
if type(d[key][item]) is types.ListType:
print_list(d[key][item],item,1)
elif type(d[key][item]) is types.DictType:
print_dict(d[key][item],item,1)
else:
if verbose:
print item,':',readability( item, d[key][item] )
else:
print item,':',d[key][item]
print '}*** END',key,'***\n'
return
if not col_fmt:
col_fmt =3D [ 'jobid', 'jobtype', 'state', 'status', 'class', 'sche=
d',=20
'client', 'server', 'start', 'elapsed', 'end', 'stunit'=
,=20
'try', 'operation', 'kbytes', 'files', 'path_last_writt=
en',
'percent', 'jobpid', 'owner', 'subtype', 'classtype',=
=20
'schedtype', 'priority', 'group', 'master_server',=20
'retention_units', 'retention_period', 'compression',=
=20
'kbyteslastwritten', 'fileslastwritten', 'filelistcount=
' ]
col_out =3D ''
for header in col_fmt:
col_out +=3D header.upper()+','
col_out =3D col_out.rstrip(',')
print col_out
for key in keys:
col_out =3D ''
for column in col_fmt:
if verbose:
data =3D readability(column, d[key][column])
col_out +=3D data+','
else:
col_out +=3D d[key][column]+','
col_out =3D col_out.rstrip(',')
print col_out
def readability( key, string ):
try:
if key =3D=3D 'jobtype':
string =3D job_type[string]
elif key =3D=3D 'state':
string =3D job_state[string]
elif key =3D=3D 'schedtype':
string =3D sched_type[string]
elif key in ['start','end','trystarted','tryended']:
if mdy:
string =3D time.strftime( '%m/%d/%Y %H:%M:%S', time.localti=
me(int(string)))
else:
string =3D time.strftime( '%d/%b/%Y %H:%M:%S', time.localti=
me(int(string)))
elif key in ['elapsed','tryelapsed']:
(h,m,s) =3D sec_to_hms(string)
string =3D '%d:%02d:%02d' % (h,m,s)
except:
pass
return string
def output_debug_dict( d ):
keys =3D d.keys()
keys.sort()
print >>sys.stderr, 'DEBUG: ', d['jobid'],'{'
for key in keys:
if type(d[key]) is types.ListType:
print_list(d[key],key,1)
elif type(d[key]) is types.DictType:
print_dict(d[key],key,1)
else:
print >>sys.stderr, 'DEBUG: ', key,':',d[key]
print >>sys.stderr, 'DEBUG: }*** END',d['jobid'],'***'
return
if __name__ =3D=3D '__main__':
try:
opts, args =3D getopt.getopt(sys.argv[1:], "f:s:e:hvxad", ["usage",=
"mdy"])
except getopt.GetoptError:
# print help information to stderr and exit:
usage()
sys.exit(2)
if not args:
for o, a in opts:
if o =3D=3D "-h":
usage()
sys.exit()
if o =3D=3D "--usage":
detailed_usage()
sys.exit()
print >>sys.stderr, '\nArgument list can not be empty'
print >>sys.stderr, 'use "-" for stdin'
usage()
sys.exit(1)
# Check to see if filenames are valid files
argflag =3D False
for arg in args:
if arg !=3D '-':
if not os.path.exists(arg):
argflag =3D True
print >>sys.stderr, '\nFile', arg, 'does not exist.'
if argflag:
usage()
sys.exit(1)
# Commandline argument defaults
all_data =3D False # -a
debug_mode =3D False # -d
xplicite =3D False # -x (not used yet)
verbose =3D False # -v
mdy =3D False # --mdy
start_date =3D 0 # -s
end_date =3D time.mktime(time.localtime()) # -e
format_file =3D '' # -f
col_fmt =3D '' # parsed column outpu=
t string
for o, a in opts:
if o =3D=3D "-h":
usage()
sys.exit()
if o =3D=3D "--usage":
detailed_usage()
sys.exit()
if o =3D=3D "-f":
format_file =3D a
if o =3D=3D "-d":
debug_mode =3D True
if o =3D=3D "-v":
verbose =3D True
if o =3D=3D "--mdy":
mdy =3D True
if o =3D=3D "-s":
try:
start_date =3D time.mktime(time.strptime(a, '%d/%b/%Y'))
except:
print >>sys.stderr, '\nDate values must be in dd/mmm/yyyy f=
ormat'
usage()
sys.exit(1)
if o =3D=3D "-e":
try:
end_date =3D time.mktime(time.strptime(a, '%d/%b/%Y'))
end_date +=3D 86399 # Add 23:59:59 to enddate to include =
that day
except:
print >>sys.stderr, '\nDate values must be in dd/mmm/yyyy f=
ormat'
usage()
sys.exit(1)
if o =3D=3D "-a":
all_data =3D True
if o =3D=3D "-x":
xplicite =3D True
d_master =3D {}
try:
if debug_mode:
print >>sys.stderr, 'DEBUG: Options and Arguments:'
for o,a in opts:
print >>sys.stderr, 'DEBUG: ', o, a
for line in csv.reader(fileinput.input(args), escapechar=3D'\\'):
try:
d, exc, buf_debug =3D process_line(line)
if exc:
raise
except:
if debug_mode:
print >>sys.stderr, 'DEBUG: Filename: ', f=
ileinput.filename()
print >>sys.stderr, 'DEBUG: Line Number: ', f=
ileinput.lineno()
print >>sys.stderr, 'DEBUG: Exception: ', e=
xc[0]
print >>sys.stderr, 'DEBUG: Exception: ', e=
xc[1]
print >>sys.stderr, 'DEBUG: Dict Contents: '
output_debug_dict( d )
print >>sys.stderr, 'DEBUG: ', buf_debug
print >>sys.stderr, 'DEBUG: ', line
print >>sys.stderr, 'DEBUG: ', '*'*30
else:
print >>sys.stderr, 'ERROR: ', line
else:
if int(d['start']) >=3D start_date and int(d['start'])<=3D =
end_date:
if int(d['state']) =3D=3D 3:
if not d_master.get(d['jobid']):
try:
d_master[d['jobid']].append(d)
except:
d_master[d['jobid']] =3D d
if format_file:
col_fmt =3D get_output_cols( format_file )
output_data( d_master, col_fmt )
except KeyboardInterrupt: # Catch premature ^C
sys.exit(3)
# modeline vim:set ts=3D4 sw=3D4 et:
--zjcmjzIkjQU2rmur--
--27ZtN5FSuKKSZcBU
Content-Type: application/pgp-signature
Content-Disposition: inline
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.2.3 (GNU/Linux)
iD8DBQE/4JKoMrO4/Yb/xwYRAumMAKDFLFGMP4U8r3QUfFaZZCY0Srj9MACfcJNX
83/3KZ/juCoS6tOxMlD/Xzk=
=hIty
-----END PGP SIGNATURE-----
--27ZtN5FSuKKSZcBU--
|
