Bacula-users

Re: [Bacula-users] Problem changing tapes due to inconsistency between 'list media' and 'query catalog'?

2010-01-28 11:45:39
Subject: Re: [Bacula-users] Problem changing tapes due to inconsistency between 'list media' and 'query catalog'?
From: Thomas Dhollander <thomas.dhollander AT gmail DOT com>
To: John Drescher <drescherjm AT gmail DOT com>
Date: Thu, 28 Jan 2010 17:42:40 +0100
Im sorry John, missed a large chunk of the files. Here they are:

director:
----------

Director {
  Name = blade03-dir
  DIRport = 9101
  QueryFile = "/usr/lib64/bacula/query.sql"
  WorkingDirectory = "/var/lib/bacula"
  PidDirectory = "/var/run"
  Maximum Concurrent Jobs = 1
  Password = "re73hjSIcRcixolfzXJsaeaJafv1hHC2juYgss4Zrx/q"
  Messages = Daemon
}

JobDefs {
  Name = DefaultJob
  Type = Backup
  Level = Incremental
  FileSet = FullClientBackup
  Schedule = WeeklyCycle
  Storage = TapeDrive
  Messages = Standard
  Pool = Default
  Priority = 10
}

JobDefs {
  Name = client-jobdefs
  Type = Backup
  Level = Incremental
  FileSet = FullClientBackup
  Schedule = "WeeklyCycle"
  Storage = TapeDrive #File
  Messages = Standard
  Pool = Default
  Priority = 10
}

#Job {
#  Name = "BackupClient1"
#  JobDefs = "DefaultJob"
#  Write Bootstrap = "/var/lib/bacula/Client1.bsr"
#}

Job {
  Name = VirtualImagesJob
  JobDefs = DefaultJob
  Client = blade03-fd
  FileSet = VirtualImagesFileSet
}

Job {
  Name = blade01-job
  JobDefs = client-jobdefs
  Client = blade01-fd
}

Job {
  Name = blade02-job
  JobDefs = client-jobdefs
  Client = blade02-fd
}

Job {
  Name = blade03-job
  JobDefs = client-jobdefs
  Client = blade03-fd
}

Job {
  Name = lindir01-job
  JobDefs = client-jobdefs
  Client = lindir01-fd
}

Job {
  Name = lindir02-job
  JobDefs = client-jobdefs
  Client = lindir02-fd
}

Job {
  Name = lingtw01-job
  JobDefs = client-jobdefs
  Client = lingtw01-fd
}

Job {
  Name = linlog01-job
  JobDefs = client-jobdefs
  Client = linlog01-fd
}

Job {
  Name = linmgt01-job
  JobDefs = client-jobdefs
  Client = linmgt01-fd
}

Job {
  Name = linmon01-job
  JobDefs = client-jobdefs
  Client = linmon01-fd
}

Job {
  Name = linnas01-job
  JobDefs = client-jobdefs
  Client = linnas01-fd
}

Job {
  Name = linscm01-job
  JobDefs = client-jobdefs
  Client = linscm01-fd
}

Job {
  Name = linsvn01-job
  JobDefs = client-jobdefs
  Client = linsvn01-fd
}

Job {
  Name = linswd01-job
  JobDefs = client-jobdefs
  Client = linswd01-fd
}

Job {
  Name = "BackupCatalog"
  JobDefs = "DefaultJob"
  Level = Full
  FileSet="Catalog"
  Client = blade03-fd
  Schedule = "WeeklyCycleAfterBackup"
  # This creates an ASCII copy of the catalog
  # WARNING!!! Passing the password via the command line is insecure.
  # see comments in make_catalog_backup for details.
  # Arguments to make_catalog_backup are:
  #  make_catalog_backup <database-name> <user-name> <password> <host>
  RunBeforeJob = "/usr/lib64/bacula/make_catalog_backup bacula bacula"
  # This deletes the copy of the catalog
  RunAfterJob  = "/usr/lib64/bacula/delete_catalog_backup"
  Write Bootstrap = "/var/lib/bacula/BackupCatalog.bsr"
  Priority = 11                   # run after main backup
}

Job {
  Name = RestoreFiles
  Type = Restore
  Client = blade03-fd
  FileSet = FullClientBackup
  Pool = Default
  Storage = TapeDrive
  Messages = Standard
  Where = /tmp/bacula-restores
}

### Used to backup Linux systems
FileSet {
  Name = FullClientBackup
  Include {
    Options { Signature = SHA1 }
    File = /
    File = /boot
    File = /dev
    File = /home
    File = /opt
    File = /usr
    File = /var
  }
  Exclude {
    File = /dev/shm
    File = /dev/pts
    File = /proc
    File = /selinux
    File = /sys
    File = /tmp
    File = /var/lib/nfs/rpc_pipefs
  }
}

### Used to backup Linux virtual images (unsafe)
FileSet {
  Name = VirtualImagesFileSet
  Include {
    Options {
      Compression = GZIP
      Signature = SHA1
      Sparse = yes
#      Regex = "^/dev/mapper/virtvg-"
#      WildFile = /dev/mapper/virtvg-*
    }
    File = /dev/mapper/virtvg-linbac01--rootlv
    File = /dev/mapper/virtvg-linbld01--rootlv
    File = /dev/mapper/virtvg-linbld02--rootlv
    File = /dev/mapper/virtvg-lindir01--rootlv
    File = /dev/mapper/virtvg-lindir02--rootlv
    File = /dev/mapper/virtvg-lingtw01--rootlv
    File = /dev/mapper/virtvg-linlog01--rootlv
    File = /dev/mapper/virtvg-linmgt01--rootlv
    File = /dev/mapper/virtvg-linmon01--rootlv
    File = /dev/mapper/virtvg-linnas01--rootlv
    File = /dev/mapper/virtvg-linscm01--rootlv
    File = /dev/mapper/virtvg-linsvn01--rootlv
    File = /dev/mapper/virtvg-linswd01--rootlv
    File = /dev/mapper/virtvg-linswd01--data01lv
  }
}

#
# When to do the backups, full backup on first sunday of the month,
#  differential (i.e. incremental since full) every other sunday,
#  and incremental backups other days
Schedule {
  Name = "WeeklyCycle"
#  Run = Full 1st sun at 23:05
#  Run = Differential 2nd-5th sun at 23:05
#  Run = Incremental mon-sat at 23:05
  Run = Full sun at 23:05
  Run = Incremental mon-sat at 23:05
}

# This schedule does the catalog. It starts after the WeeklyCycle
Schedule {
  Name = "WeeklyCycleAfterBackup"
  Run = Full mon-sun at 6:00
}

# This is the backup of the catalog
FileSet {
  Name = "Catalog"
  Include {
    Options {
      signature = MD5
    }
    File = /var/lib/bacula/bacula.sql
  }
}

Client {
  Name = blade01-fd
  Address = blade01.dsquare.intra.
  Catalog = ClientDB
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
}

Client {
  Name = blade02-fd
  Address = blade02.dsquare.intra.
  Catalog = ClientDB
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
}

Client {
  Name = blade03-fd
  Address = blade03.dsquare.intra.
  Catalog = ClientDB
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
}

Client {
  Name = lindir01-fd
  Address = lindir01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = lindir02-fd
  Address = lindir02.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = lingtw01-fd
  Address = lingtw01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linlog01-fd
  Address = linlog01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linmgt01-fd
  Address = linmgt01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linmon01-fd
  Address = linmon01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linnas01-fd
  Address = linnas01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linscm01-fd
  Address = linscm01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linsvn01-fd
  Address = linsvn01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

Client {
  Name = linswd01-fd
  Address = linswd01.dsquare.intra.
  Password = "qIrfIo8I75DC7hwGdrZvNACuiVnGiVzK/jB/nHorhpmH"
  Catalog = ClientDB
}

# Definition of "Tape Drive" storage device
Storage {
  Name = TapeDrive
  Address = blade03.dsquare.intra.               # N.B. Use a fully qualified 
name here
  SDPort = 9103
  Password = "Vzl7x+Z+WEFvLNXRip1XWMKZRT8gGLhbT/N1Z+moemxv"
  Device = TS2900
  Media Type = LTO-4
}

# Generic catalog service
Catalog {
  Name = ClientDB
# Uncomment the following line if you want the dbi driver
# dbdriver = "dbi:sqlite3"; dbaddress = 127.0.0.1; dbport =  
  dbname = "bacula"; dbuser = "bacula"; dbpassword = ""
}

# Reasonable message delivery -- send most everything to email address
#  and to the console
Messages {
  Name = Standard
#
# NOTE! If you send to two email or more email addresses, you will need
#  to replace the %r in the from field (-f part) with a single valid
#  email address in both the mailcommand and the operatorcommand.
#  What this does is, it sets the email address that emails would display
#  in the FROM field, which is by default the same email as they're being
#  sent to.  However, if you send email to more than one address, then
#  you'll have to set the FROM address manually, to a single address. 
#  for example, a 'no-reply AT mydomain DOT com', is better since that tends to
#  tell (most) people that its coming from an automated source.

#
  mailcommand = "/usr/sbin/bsmtp -h localhost -f \"\(Bacula\) \<%r\>\" -s 
\"Bacula: %t %e of %c %l\" %r"
  operatorcommand = "/usr/sbin/bsmtp -h localhost -f \"\(Bacula\) \<%r\>\" -s 
\"Bacula: Intervention needed 
for %j\" %r"
  mail = root@localhost = all, !skipped            
  operator = root@localhost = mount
  console = all, !skipped, !saved
#
# WARNING! the following will create a file that you must cycle from
#          time to time as it will grow indefinitely. However, it will
#          also keep all your messages if they scroll off the console.
#
  append = "/var/lib/bacula/log" = all, !skipped
  syslog = all, !skipped
  catalog = all
}

#
# Message delivery for daemon messages (no job).
Messages {
  Name = Daemon
  mailcommand = "/usr/sbin/bsmtp -h localhost -f \"\(Bacula\) \<%r\>\" -s 
\"Bacula daemon message\" %r"
  mail = root@localhost = all, !skipped            
  console = all, !skipped, !saved
  syslog = all, !skipped
}

# Default pool definition
Pool {
  Name = Default
  Pool Type = Backup
  Recycle = yes                       # Bacula can automatically recycle Volumes
  AutoPrune = yes                     # Prune expired volumes
  Volume Retention = 365 days         # one year
  Label Type = IBM
}


storage daemon:
----------------------

Storage {                             # definition of myself
  Name = blade03-sd
  SDPort = 9103                  # Director's port      
  WorkingDirectory = "/var/lib/bacula"
  Pid Directory = "/var/run"
  Maximum Concurrent Jobs = 20
}

Director {
  Name = blade03-dir
  Password = "Vzl7x+Z+WEFvLNXRip1XWMKZRT8gGLhbT/N1Z+moemxv"
}

Autochanger{
  Name = Autochanger
  Device = TS2900
  Changer Command = "/usr/lib64/bacula/mtx-changer %c %o %S %a %d"
  Changer Device = /dev/changer
}

Device {
  Name = TS2900
  Drive Index = 0
  Media Type = LTO-4
#  Archive Device = /dev/IBMtape0n
  Archive Device = /dev/nst0
  AutomaticMount = yes               # when device opened, read it
  Autoselect = yes
  AlwaysOpen = yes
  RemovableMedia = yes
  RandomAccess = no
  AutoChanger = yes
  #Alert Command = "sh -c 'tapeinfo -f %c | grep TapeAlert | cat'"
  Maximum Network Buffer Size = 65536
  Spool Directory = /var/spool/bacula
  Label Type = IBM
  Check Labels = yes
}

Messages {
  Name = Standard
  director = blade03-dir = all
}


On 28 Jan 2010, at 16:36, John Drescher wrote:

> On Thu, Jan 28, 2010 at 10:32 AM, Thomas Dhollander
> <thomas.dhollander AT gmail DOT com> wrote:
>> Hi John
>> 
>> Thanks for your reaction. I added the conf files below.
>> 
>> Looking forward to any hints / thoughts on this.
>> 
>> Thomas
>> 
>> 
>> On 28 Jan 2010, at 14:42, John Drescher wrote:
>> 
>>>> In our current setup, Bacula does not seem to automatically load the 
>>>> correct
>>>> tape in the drive for its backups. We did specify the correct autochanger
>>>> options in the storage daemon conf file and made sure the device uses the
>>>> autochanger.
>>>> 
>>>> When we execute the 'list media' command in bconsole, we get the following
>>>> output:
>>>> 
>>>> Automatically selected Catalog: ClientDB
>>>> Using Catalog "ClientDB"
>>>> Pool: Default
>>>> +---------+------------+-----------+---------+----------------+----------+--------------+---------+------+-----------+-----------+---------------------+
>>>> | MediaId | VolumeName | VolStatus | Enabled | VolBytes       | VolFiles |
>>>> VolRetention | Recycle | Slot | InChanger | MediaType | LastWritten
>>>> |
>>>> +---------+------------+-----------+---------+----------------+----------+--------------+---------+------+-----------+-----------+---------------------+
>>>> |       1 | 426BBL     | Append    |       1 | 26,842,862,592 |       65 |
>>>> 31,536,000 |       1 |    0 |         1 | LTO-4     | 2010-01-28 06:00:11 |
>>>> +---------+------------+-----------+---------+----------------+----------+--------------+---------+------+-----------+-----------+---------------------+
>>>> 
>>>> from which i derive that Bacula recognizes one pool ('Default') with one
>>>> associated volume, that is currently in the autochanger.
>>>> 
>>>> But the 'query catalog' command (we use the mysql db) seems not to know
>>>> about the volume in the autochanger:
>>>> 
>>>> *query catalog
>>>> Automatically selected Catalog: ClientDB
>>>> Using Catalog "ClientDB"
>>>> Available queries:
>>>>      1: List up to 20 places where a File is saved regardless of the
>>>> directory
>>>>      2: List where the most recent copies of a file are saved
>>>>      3: List last 20 Full Backups for a Client
>>>>      4: List all backups for a Client after a specified time
>>>>      5: List all backups for a Client
>>>>      6: List Volume Attributes for a selected Volume
>>>>      7: List Volumes used by selected JobId
>>>>      8: List Volumes to Restore All Files
>>>>      9: List Pool Attributes for a selected Pool
>>>>     10: List total files/bytes by Job
>>>>     11: List total files/bytes by Volume
>>>>     12: List Files for a selected JobId
>>>>     13: List Jobs stored on a selected MediaId
>>>>     14: List Jobs stored for a given Volume name
>>>>     15: List Volumes Bacula thinks are in changer
>>>>     16: List Volumes likely to need replacement from age or errors
>>>> Choose a query (1-16): 15
>>>> No results to list.
>>>> 
>>>> In addition, the logs show the following message blocks:
>>>> 28-Jan 04:00 blade03-sd JobId 19: Invalid slot=0 defined in catalog for
>>>> Volume "426BBL" on "TS2900" (/dev/nst0). Manual load may be required.
>>>> 28-Jan 04:00 blade03-sd JobId 19: 3301 Issuing autochanger "loaded? drive 
>>>> 0"
>>>> command.
>>>> 28-Jan 04:00 blade03-sd JobId 19: 3302 Autochanger "loaded? drive 0",
>>>> result: nothing loaded.
>>>> 28-Jan 04:00 blade03-sd JobId 19: 3301 Issuing autochanger "loaded? drive 
>>>> 0"
>>>> command.
>>>> 28-Jan 04:00 blade03-sd JobId 19: 3302 Autochanger "loaded? drive 0",
>>>> result: nothing loaded.
>>>> 28-Jan 04:00 blade03-sd JobId 19: Warning: mount.c:219 Open device "TS2900"
>>>> (/dev/nst0) Volume "426BBL" failed: ERR=dev.c:474 Unable to open device
>>>> "TS2900" (/dev/nst0): ERR=No medium found
>>>> 
>>>> In other words, the catalog does not seem to know about the volumes in the
>>>> autochanger slots? Could we run a bscan or take other actions to restore
>>>> consistency between the catalog and the tape library? We already tried the
>>>> 'update slots' command but that had no effect.
>>>> 
>>>> Note that we are just getting started with Bacula soo re-initializing the
>>>> whole catalog is an option if it would make our setup consistent again.
>>>> 
>>> That would require you to erase each tape used with mt.
>>> 
>>> Can you post your bacula-sd.conf and bacula-dir.conf and your bacula 
>>> version.
>> 
>> bacula-sd.conf:
>> --------------------
>> 
>> Storage {                             # definition of myself
>>  Name = blade03-sd
>>  SDPort = 9103                  # Director's port
>>  WorkingDirectory = "/var/lib/bacula"
>>  Pid Directory = "/var/run"
>>  Maximum Concurrent Jobs = 20
>> }
>> 
>> Director {
>>  Name = blade03-dir
>>  Password = "Vzl7x+Z+WEFvLNXRip1XWMKZRT8gGLhbT/N1Z+moemxv"
>> }
>> 
>> Autochanger{
>>  Name = Autochanger
>>  Device = TS2900
>>  Changer Command = "/usr/lib64/bacula/mtx-changer %c %o %S %a %d"
>>  Changer Device = /dev/changer
>> }
>> 
>> Device {
>>  Name = TS2900
>>  Drive Index = 0
>>  Media Type = LTO-4
>> #  Archive Device = /dev/IBMtape0n
>>  Archive Device = /dev/nst0
>>  AutomaticMount = yes               # when device opened, read it
>>  Autoselect = yes
>>  AlwaysOpen = yes
>>  RemovableMedia = yes
>>  RandomAccess = no
>>  AutoChanger = yes
>>  #Alert Command = "sh -c 'tapeinfo -f %c | grep TapeAlert | cat'"
>>  Maximum Network Buffer Size = 65536
>>  Spool Directory = /var/spool/bacula
>>  Label Type = IBM
>>  Check Labels = yes
>> }
>> 
>> Messages {
>>  Name = Standard
>>  director = blade03-dir = all
>> }
>> 
>> bacula-dir.conf:
>> --------------------
>> 
>> # Bacula
>> Director {
>>  Name = blade03-dir
>>  DIRport = 9101
>>  QueryFile = "/usr/lib64/bacula/query.sql"
>>  WorkingDirectory = "/var/lib/bacula"
>>  PidDirectory = "/var/run"
>>  Maximum Concurrent Jobs = 1
>>  Password = "re73hjSIcRcixolfzXJsaeaJafv1hHC2juYgss4Zrx/q"
>>  Messages = Daemon
>> }
>> 
>> JobDefs {
>>  Name = DefaultJob
>>  Type = Backup
>>  Level = Incremental
>>  FileSet = FullClientBackup
>>  Schedule = WeeklyCycle
>>  Storage = TapeDrive
>>  Messages = Standard
>>  Pool = Default
>>  Priority = 10
>> }
>> 
>> JobDefs {
>>  Name = client-jobdefs
>>  Type = Backup
>>  Level = Incremental
>>  FileSet = FullClientBackup
>>  Schedule = "WeeklyCycle"
>>  Storage = TapeDrive #File
>>  Messages = Standard
>>  Pool = Default
>>  Priority = 10
>> }
>> 
>> #Job {
>> #  Name = "BackupClient1"
>> #  JobDefs = "DefaultJob"
>> #  Write Bootstrap = "/var/lib/bacula/Client1.bsr"
>> #}
>> 
>> Job {
>>  Name = VirtualImagesJob
>>  JobDefs = DefaultJob
>>  Client = blade03-fd
>>  FileSet = VirtualImagesFileSet
>> }
>> 
> 
> You are missing the storage and client sections of bacula-dir.conf.
> The stroage section probably is enough.
> 
> John


------------------------------------------------------------------------------
The Planet: dedicated and managed hosting, cloud storage, colocation
Stay online with enterprise data centers and the best network in the business
Choose flexible plans and management services without long-term contracts
Personal 24x7 support from experience hosting pros just a phone call away.
http://p.sf.net/sfu/theplanet-com
_______________________________________________
Bacula-users mailing list
Bacula-users AT lists.sourceforge DOT net
https://lists.sourceforge.net/lists/listinfo/bacula-users