| 1 |
""" pyrobocopy.py -
|
| 2 |
|
| 3 |
Version: 1.0
|
| 4 |
|
| 5 |
Report the difference in content
|
| 6 |
of two directories, synchronize or
|
| 7 |
update a directory from another, taking
|
| 8 |
into account time-stamps of files etc.
|
| 9 |
|
| 10 |
By Anand B Pillai
|
| 11 |
|
| 12 |
(This program is inspired by the windows
|
| 13 |
'Robocopy' program.)
|
| 14 |
|
| 15 |
Mod Nov 11 Rewrote to use the filecmp module.
|
| 16 |
"""
|
| 17 |
|
| 18 |
import os, stat
|
| 19 |
import time
|
| 20 |
import shutil
|
| 21 |
import filecmp
|
| 22 |
|
| 23 |
def usage():
|
| 24 |
return """
|
| 25 |
Pyrobocopy: Command line directory diff, synchronization, update & copy
|
| 26 |
|
| 27 |
Author: Anand Pillai
|
| 28 |
|
| 29 |
Usage: %s <sourcedir> <targetdir> Options
|
| 30 |
|
| 31 |
Main Options:\n
|
| 32 |
\t-d --diff - Only report difference between sourcedir and targetdir
|
| 33 |
\t-s, --synchronize - Synchronize content between sourcedir and targetdir
|
| 34 |
\t-u, --update - Update existing content between sourcedir and targetdir
|
| 35 |
|
| 36 |
Additional Options:\n
|
| 37 |
\t-p, --purge - Purge files when synchronizing (does not purge by default).
|
| 38 |
\t-f, --force - Force copying of files, by trying to change file permissions.
|
| 39 |
\t-n, --nodirection - Update files in source directory from target
|
| 40 |
\t directory (only updates target from source by default).
|
| 41 |
\t-c, --create - Create target directory if it does not exist (By default,
|
| 42 |
\t target directory should exist.)
|
| 43 |
\t-m, --modtime - Only compare file's modification times for an update (By default,
|
| 44 |
\t compares source file's creation time also).
|
| 45 |
"""
|
| 46 |
|
| 47 |
|
| 48 |
|
| 49 |
class PyRobocopier:
|
| 50 |
""" An advanced directory synchronization, updation
|
| 51 |
and file copying class """
|
| 52 |
|
| 53 |
prog_name = "pyrobocopy.py"
|
| 54 |
|
| 55 |
def __init__(self):
|
| 56 |
|
| 57 |
self.__dir1 = ''
|
| 58 |
self.__dir2 = ''
|
| 59 |
self.__dcmp = None
|
| 60 |
|
| 61 |
self.__copyfiles = True
|
| 62 |
self.__forcecopy = False
|
| 63 |
self.__copydirection = 0
|
| 64 |
self.__updatefiles = True
|
| 65 |
self.__creatdirs = True
|
| 66 |
self.__purge =False
|
| 67 |
self.__maketarget =False
|
| 68 |
self.__modtimeonly =False
|
| 69 |
self.__mainfunc = None
|
| 70 |
|
| 71 |
# stat vars
|
| 72 |
self.__numdirs =0
|
| 73 |
self.__numfiles =0
|
| 74 |
self.__numdelfiles =0
|
| 75 |
self.__numdeldirs =0
|
| 76 |
self.__numnewdirs =0
|
| 77 |
self.__numupdates =0
|
| 78 |
self.__starttime = 0.0
|
| 79 |
self.__endtime = 0.0
|
| 80 |
|
| 81 |
# failure stat vars
|
| 82 |
self.__numcopyfld =0
|
| 83 |
self.__numupdsfld =0
|
| 84 |
self.__numdirsfld =0
|
| 85 |
self.__numdelffld =0
|
| 86 |
self.__numdeldfld =0
|
| 87 |
|
| 88 |
def parse_args(self, arguments):
|
| 89 |
""" Parse arguments """
|
| 90 |
|
| 91 |
import getopt
|
| 92 |
|
| 93 |
shortargs = "supncm"
|
| 94 |
longargs = ["synchronize=", "update=", "purge=", "nodirection=", "create=", "modtime="]
|
| 95 |
|
| 96 |
try:
|
| 97 |
optlist, args = getopt.getopt( arguments, shortargs, longargs )
|
| 98 |
except getopt.GetoptError, e:
|
| 99 |
print e
|
| 100 |
return None
|
| 101 |
|
| 102 |
allargs = []
|
| 103 |
if len(optlist):
|
| 104 |
allargs = [x[0] for x in optlist]
|
| 105 |
|
| 106 |
allargs.extend( args )
|
| 107 |
self.__setargs( allargs )
|
| 108 |
|
| 109 |
def __setargs(self, argslist):
|
| 110 |
""" Sets internal variables using arguments """
|
| 111 |
|
| 112 |
for option in argslist:
|
| 113 |
if option.lower() in ('-s', '--synchronize'):
|
| 114 |
self.__mainfunc = self.synchronize
|
| 115 |
elif option.lower() in ('-u', '--update'):
|
| 116 |
self.__mainfunc = self.update
|
| 117 |
elif option.lower() in ('-d', '--diff'):
|
| 118 |
self.__mainfunc = self.dirdiff
|
| 119 |
elif option.lower() in ('-p', '--purge'):
|
| 120 |
self.__purge = True
|
| 121 |
elif option.lower() in ('-n', '--nodirection'):
|
| 122 |
self.__copydirection = 2
|
| 123 |
elif option.lower() in ('-f', '--force'):
|
| 124 |
self.__forcecopy = True
|
| 125 |
elif option.lower() in ('-c', '--create'):
|
| 126 |
self.__maketarget = True
|
| 127 |
elif option.lower() in ('-m', '--modtime'):
|
| 128 |
self.__modtimeonly = True
|
| 129 |
else:
|
| 130 |
if self.__dir1=='':
|
| 131 |
self.__dir1 = option
|
| 132 |
elif self.__dir2=='':
|
| 133 |
self.__dir2 = option
|
| 134 |
|
| 135 |
if self.__dir1=='' or self.__dir2=='':
|
| 136 |
sys.exit("Argument Error: Directory arguments not given!")
|
| 137 |
if not os.path.isdir(self.__dir1):
|
| 138 |
sys.exit("Argument Error: Source directory does not exist!")
|
| 139 |
if not self.__maketarget and not os.path.isdir(self.__dir2):
|
| 140 |
sys.exit("Argument Error: Target directory %s does not exist! (Try the -c option)." % self.__dir2)
|
| 141 |
if self.__mainfunc is None:
|
| 142 |
sys.exit("Argument Error: Specify an action (Diff, Synchronize or Update) ")
|
| 143 |
|
| 144 |
self.__dcmp = filecmp.dircmp(self.__dir1, self.__dir2)
|
| 145 |
|
| 146 |
def do_work(self):
|
| 147 |
""" Do work """
|
| 148 |
|
| 149 |
self.__starttime = time.time()
|
| 150 |
|
| 151 |
if not os.path.isdir(self.__dir2):
|
| 152 |
if self.__maketarget:
|
| 153 |
print 'Creating directory', self.__dir2
|
| 154 |
try:
|
| 155 |
os.makedirs(self.__dir2)
|
| 156 |
except Exception, e:
|
| 157 |
print e
|
| 158 |
return None
|
| 159 |
|
| 160 |
# All right!
|
| 161 |
self.__mainfunc()
|
| 162 |
self.__endtime = time.time()
|
| 163 |
|
| 164 |
def __dowork(self, dir1, dir2, copyfunc = None, updatefunc = None):
|
| 165 |
""" Private attribute for doing work """
|
| 166 |
|
| 167 |
print 'Source directory: ', dir1, ':'
|
| 168 |
|
| 169 |
self.__numdirs += 1
|
| 170 |
self.__dcmp = filecmp.dircmp(dir1, dir2)
|
| 171 |
|
| 172 |
# Files & directories only in target directory
|
| 173 |
if self.__purge:
|
| 174 |
for f2 in self.__dcmp.right_only:
|
| 175 |
fullf2 = os.path.join(dir2, f2)
|
| 176 |
print 'Deleting ',fullf2
|
| 177 |
try:
|
| 178 |
if os.path.isfile(fullf2):
|
| 179 |
|
| 180 |
try:
|
| 181 |
os.remove(fullf2)
|
| 182 |
self.__numdelfiles += 1
|
| 183 |
except OSError, e:
|
| 184 |
print e
|
| 185 |
self.__numdelffld += 1
|
| 186 |
elif os.path.isdir(fullf2):
|
| 187 |
try:
|
| 188 |
shutil.rmtree( fullf2, True )
|
| 189 |
self.__numdeldirs += 1
|
| 190 |
except shutil.Error, e:
|
| 191 |
print e
|
| 192 |
self.__numdeldfld += 1
|
| 193 |
|
| 194 |
except Exception, e: # of any use ?
|
| 195 |
print e
|
| 196 |
continue
|
| 197 |
|
| 198 |
|
| 199 |
# Files & directories only in source directory
|
| 200 |
for f1 in self.__dcmp.left_only:
|
| 201 |
try:
|
| 202 |
st = os.stat(os.path.join(dir1, f1))
|
| 203 |
except os.error:
|
| 204 |
continue
|
| 205 |
|
| 206 |
if stat.S_ISREG(st.st_mode):
|
| 207 |
if copyfunc: copyfunc(f1, dir1, dir2)
|
| 208 |
elif stat.S_ISDIR(st.st_mode):
|
| 209 |
fulld1 = os.path.join(dir1, f1)
|
| 210 |
fulld2 = os.path.join(dir2, f1)
|
| 211 |
|
| 212 |
if self.__creatdirs:
|
| 213 |
try:
|
| 214 |
# Copy tree
|
| 215 |
print 'Copying tree', fulld2
|
| 216 |
shutil.copytree(fulld1, fulld2)
|
| 217 |
self.__numnewdirs += 1
|
| 218 |
print 'Done.'
|
| 219 |
except shutil.Error, e:
|
| 220 |
print e
|
| 221 |
self.__numdirsfld += 1
|
| 222 |
|
| 223 |
# jump to next file/dir in loop since this op failed
|
| 224 |
continue
|
| 225 |
|
| 226 |
# Call tail recursive
|
| 227 |
# if os.path.exists(fulld2):
|
| 228 |
# self.__dowork(fulld1, fulld2, copyfunc, updatefunc)
|
| 229 |
|
| 230 |
# common files/directories
|
| 231 |
for f1 in self.__dcmp.common:
|
| 232 |
try:
|
| 233 |
st = os.stat(os.path.join(dir1, f1))
|
| 234 |
except os.error:
|
| 235 |
continue
|
| 236 |
|
| 237 |
if stat.S_ISREG(st.st_mode):
|
| 238 |
if updatefunc: updatefunc(f1, dir1, dir2)
|
| 239 |
elif stat.S_ISDIR(st.st_mode):
|
| 240 |
fulld1 = os.path.join(dir1, f1)
|
| 241 |
fulld2 = os.path.join(dir2, f1)
|
| 242 |
# Call tail recursive
|
| 243 |
self.__dowork(fulld1, fulld2, copyfunc, updatefunc)
|
| 244 |
|
| 245 |
|
| 246 |
def __copy(self, filename, dir1, dir2):
|
| 247 |
""" Private function for copying a file """
|
| 248 |
|
| 249 |
# NOTE: dir1 is source & dir2 is target
|
| 250 |
if self.__copyfiles:
|
| 251 |
|
| 252 |
print 'Copying file', filename, dir1, dir2
|
| 253 |
try:
|
| 254 |
if self.__copydirection== 0 or self.__copydirection == 2: # source to target
|
| 255 |
|
| 256 |
if not os.path.exists(dir2):
|
| 257 |
if self.__forcecopy:
|
| 258 |
os.chmod(os.path.dirname(dir2), 0777)
|
| 259 |
try:
|
| 260 |
os.makedirs(dir1)
|
| 261 |
except OSError, e:
|
| 262 |
print e
|
| 263 |
self.__numdirsfld += 1
|
| 264 |
|
| 265 |
if self.__forcecopy:
|
| 266 |
os.chmod(dir2, 0777)
|
| 267 |
|
| 268 |
sourcefile = os.path.join(dir1, filename)
|
| 269 |
try:
|
| 270 |
shutil.copy(sourcefile, dir2)
|
| 271 |
self.__numfiles += 1
|
| 272 |
except (IOError, OSError), e:
|
| 273 |
print e
|
| 274 |
self.__numcopyfld += 1
|
| 275 |
|
| 276 |
elif self.__copydirection==1 or self.__copydirection == 2: # target to source
|
| 277 |
|
| 278 |
if not os.path.exists(dir1):
|
| 279 |
if self.__forcecopy:
|
| 280 |
os.chmod(os.path.dirname(dir1), 0777)
|
| 281 |
|
| 282 |
try:
|
| 283 |
os.makedirs(dir1)
|
| 284 |
except OSError, e:
|
| 285 |
print e
|
| 286 |
self.__numdirsfld += 1
|
| 287 |
|
| 288 |
targetfile = os.path.abspath(os.path.join(dir1, filename))
|
| 289 |
if self.__forcecopy:
|
| 290 |
os.chmod(dir1, 0777)
|
| 291 |
|
| 292 |
sourcefile = os.path.join(dir2, filename)
|
| 293 |
|
| 294 |
try:
|
| 295 |
shutil.copy(sourcefile, dir1)
|
| 296 |
self.__numfiles += 1
|
| 297 |
except (IOError, OSError), e:
|
| 298 |
print e
|
| 299 |
self.__numcopyfld += 1
|
| 300 |
|
| 301 |
except Exception, e:
|
| 302 |
print 'Error copying file', filename, e
|
| 303 |
|
| 304 |
def __cmptimestamps(self, filest1, filest2):
|
| 305 |
""" Compare time stamps of two files and return True
|
| 306 |
if file1 (source) is more recent than file2 (target) """
|
| 307 |
|
| 308 |
return ((filest1.st_mtime > filest2.st_mtime) or \
|
| 309 |
(not self.__modtimeonly and (filest1.st_ctime > filest2.st_mtime)))
|
| 310 |
|
| 311 |
def __update(self, filename, dir1, dir2):
|
| 312 |
""" Private function for updating a file based on
|
| 313 |
last time stamp of modification """
|
| 314 |
|
| 315 |
print 'Updating file', filename
|
| 316 |
|
| 317 |
# NOTE: dir1 is source & dir2 is target
|
| 318 |
if self.__updatefiles:
|
| 319 |
|
| 320 |
file1 = os.path.join(dir1, filename)
|
| 321 |
file2 = os.path.join(dir2, filename)
|
| 322 |
|
| 323 |
try:
|
| 324 |
st1 = os.stat(file1)
|
| 325 |
st2 = os.stat(file2)
|
| 326 |
except os.error:
|
| 327 |
return -1
|
| 328 |
|
| 329 |
# Update will update in both directions depending
|
| 330 |
# on the timestamp of the file & copy-direction.
|
| 331 |
|
| 332 |
if self.__copydirection==0 or self.__copydirection == 2:
|
| 333 |
|
| 334 |
# Update file if file's modification time is older than
|
| 335 |
# source file's modification time, or creation time. Sometimes
|
| 336 |
# it so happens that a file's creation time is newer than it's
|
| 337 |
# modification time! (Seen this on windows)
|
| 338 |
if self.__cmptimestamps( st1, st2 ):
|
| 339 |
print 'Updating file ', file2 # source to target
|
| 340 |
try:
|
| 341 |
if self.__forcecopy:
|
| 342 |
os.chmod(file2, 0666)
|
| 343 |
|
| 344 |
try:
|
| 345 |
shutil.copy(file1, file2)
|
| 346 |
self.__numupdates += 1
|
| 347 |
return 0
|
| 348 |
except (IOError, OSError), e:
|
| 349 |
print e
|
| 350 |
self.__numupdsfld += 1
|
| 351 |
return -1
|
| 352 |
|
| 353 |
except Exception, e:
|
| 354 |
print e
|
| 355 |
return -1
|
| 356 |
|
| 357 |
elif self.__copydirection==1 or self.__copydirection == 2:
|
| 358 |
|
| 359 |
# Update file if file's modification time is older than
|
| 360 |
# source file's modification time, or creation time. Sometimes
|
| 361 |
# it so happens that a file's creation time is newer than it's
|
| 362 |
# modification time! (Seen this on windows)
|
| 363 |
if self.__cmptimestamps( st2, st1 ):
|
| 364 |
print 'Updating file ', file1 # target to source
|
| 365 |
try:
|
| 366 |
if self.__forcecopy:
|
| 367 |
os.chmod(file1, 0666)
|
| 368 |
|
| 369 |
try:
|
| 370 |
shutil.copy(file2, file1)
|
| 371 |
self.__numupdates += 1
|
| 372 |
return 0
|
| 373 |
except (IOError, OSError), e:
|
| 374 |
print e
|
| 375 |
self.__numupdsfld += 1
|
| 376 |
return -1
|
| 377 |
|
| 378 |
except Exception, e:
|
| 379 |
print e
|
| 380 |
return -1
|
| 381 |
|
| 382 |
return -1
|
| 383 |
|
| 384 |
def __dirdiffandcopy(self, dir1, dir2):
|
| 385 |
""" Private function which does directory diff & copy """
|
| 386 |
self.__dowork(dir1, dir2, self.__copy)
|
| 387 |
|
| 388 |
def __dirdiffandupdate(self, dir1, dir2):
|
| 389 |
""" Private function which does directory diff & update """
|
| 390 |
self.__dowork(dir1, dir2, None, self.__update)
|
| 391 |
|
| 392 |
def __dirdiffcopyandupdate(self, dir1, dir2):
|
| 393 |
""" Private function which does directory diff, copy and update (synchro) """
|
| 394 |
self.__dowork(dir1, dir2, self.__copy, self.__update)
|
| 395 |
|
| 396 |
def __dirdiff(self):
|
| 397 |
""" Private function which only does directory diff """
|
| 398 |
|
| 399 |
if self.__dcmp.left_only:
|
| 400 |
print 'Only in', self.__dir1
|
| 401 |
for x in self.__dcmp.left_only:
|
| 402 |
print '>>', x
|
| 403 |
|
| 404 |
if self.__dcmp.right_only:
|
| 405 |
print 'Only in', self.__dir2
|
| 406 |
for x in self.__dcmp.right_only:
|
| 407 |
print '<<', x
|
| 408 |
|
| 409 |
if self.__dcmp.common:
|
| 410 |
print 'Common to', self.__dir1,' and ',self.__dir2
|
| 411 |
print
|
| 412 |
for x in self.__dcmp.common:
|
| 413 |
print '--', x
|
| 414 |
else:
|
| 415 |
print 'No common files or sub-directories!'
|
| 416 |
|
| 417 |
def synchronize(self):
|
| 418 |
""" Synchronize will try to synchronize two directories w.r.t
|
| 419 |
each other's contents, copying files if necessary from source
|
| 420 |
to target, and creating directories if necessary. If the optional
|
| 421 |
argument purge is True, directories in target (dir2) that are
|
| 422 |
not present in the source (dir1) will be deleted . Synchronization
|
| 423 |
is done in the direction of source to target """
|
| 424 |
|
| 425 |
self.__copyfiles = True
|
| 426 |
self.__updatefiles = True
|
| 427 |
self.__creatdirs = True
|
| 428 |
self.__copydirection = 0
|
| 429 |
|
| 430 |
print 'Synchronizing directory', self.__dir2, 'with', self.__dir1 ,'\n'
|
| 431 |
self.__dirdiffcopyandupdate(self.__dir1, self.__dir2)
|
| 432 |
|
| 433 |
def update(self):
|
| 434 |
""" Update will try to update the target directory
|
| 435 |
w.r.t source directory. Only files that are common
|
| 436 |
to both directories will be updated, no new files
|
| 437 |
or directories are created """
|
| 438 |
|
| 439 |
self.__copyfiles = False
|
| 440 |
self.__updatefiles = True
|
| 441 |
self.__purge = False
|
| 442 |
self.__creatdirs = False
|
| 443 |
|
| 444 |
print 'Updating directory', self.__dir2, 'from', self.__dir1 , '\n'
|
| 445 |
self.__dirdiffandupdate(self.__dir1, self.__dir2)
|
| 446 |
|
| 447 |
def dirdiff(self):
|
| 448 |
""" Only report difference in content between two
|
| 449 |
directories """
|
| 450 |
|
| 451 |
self.__copyfiles = False
|
| 452 |
self.__updatefiles = False
|
| 453 |
self.__purge = False
|
| 454 |
self.__creatdirs = False
|
| 455 |
self.__updatefiles = False
|
| 456 |
|
| 457 |
print 'Difference of directory ', self.__dir2, 'from', self.__dir1 , '\n'
|
| 458 |
self.__dirdiff()
|
| 459 |
|
| 460 |
def report(self):
|
| 461 |
""" Print report of work at the end """
|
| 462 |
|
| 463 |
# We need only the first 4 significant digits
|
| 464 |
tt = (str(self.__endtime - self.__starttime))[:4]
|
| 465 |
|
| 466 |
print '\nPython robocopier finished in',tt, 'seconds.'
|
| 467 |
print self.__numdirs, 'directories parsed,',self.__numfiles, 'files copied.'
|
| 468 |
if self.__numdelfiles:
|
| 469 |
print self.__numdelfiles, 'files were purged.'
|
| 470 |
if self.__numdeldirs:
|
| 471 |
print self.__numdeldirs, 'directories were purged.'
|
| 472 |
if self.__numnewdirs:
|
| 473 |
print self.__numnewdirs, 'directories were created.'
|
| 474 |
if self.__numupdates:
|
| 475 |
print self.__numupdates, 'files were updated by timestamp.'
|
| 476 |
|
| 477 |
# Failure stats
|
| 478 |
print '\n'
|
| 479 |
if self.__numcopyfld:
|
| 480 |
print self.__numcopyfld, 'files could not be copied.'
|
| 481 |
if self.__numdirsfld:
|
| 482 |
print self.__numdirsfld, 'directories could not be created.'
|
| 483 |
if self.__numupdsfld:
|
| 484 |
print self.__numupdsfld, 'files could not be updated.'
|
| 485 |
if self.__numdeldfld:
|
| 486 |
print self.__numdeldfld, 'directories could not be purged.'
|
| 487 |
if self.__numdelffld:
|
| 488 |
print self.__numdelffld, 'files could not be purged.'
|
| 489 |
|
| 490 |
if __name__=="__main__":
|
| 491 |
import sys
|
| 492 |
|
| 493 |
if len(sys.argv)<2:
|
| 494 |
sys.exit( usage() % PyRobocopier.prog_name )
|
| 495 |
|
| 496 |
copier = PyRobocopier()
|
| 497 |
copier.parse_args(sys.argv[1:])
|
| 498 |
copier.do_work()
|
| 499 |
|
| 500 |
# print report at the end
|
| 501 |
copier.report()
|
