ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/ns_dev/Python/NinoCode/Active_prgs/pyrobocopy.py
Revision: 8
Committed: Sat May 5 04:21:19 2012 UTC (13 years, 10 months ago) by ninoborges
Content type: text/x-python
File size: 18583 byte(s)
Log Message:
Initial Import

File Contents

# User Rev Content
1 ninoborges 8 """ pyrobocopy.py -
2    
3     Version: 1.0
4    
5     Report the difference in content
6     of two directories, synchronize or
7     update a directory from another, taking
8     into account time-stamps of files etc.
9    
10     By Anand B Pillai
11    
12     (This program is inspired by the windows
13     'Robocopy' program.)
14    
15     Mod Nov 11 Rewrote to use the filecmp module.
16     """
17    
18     import os, stat
19     import time
20     import shutil
21     import filecmp
22    
23     def usage():
24     return """
25     Pyrobocopy: Command line directory diff, synchronization, update & copy
26    
27     Author: Anand Pillai
28    
29     Usage: %s <sourcedir> <targetdir> Options
30    
31     Main Options:\n
32     \t-d --diff - Only report difference between sourcedir and targetdir
33     \t-s, --synchronize - Synchronize content between sourcedir and targetdir
34     \t-u, --update - Update existing content between sourcedir and targetdir
35    
36     Additional Options:\n
37     \t-p, --purge - Purge files when synchronizing (does not purge by default).
38     \t-f, --force - Force copying of files, by trying to change file permissions.
39     \t-n, --nodirection - Update files in source directory from target
40     \t directory (only updates target from source by default).
41     \t-c, --create - Create target directory if it does not exist (By default,
42     \t target directory should exist.)
43     \t-m, --modtime - Only compare file's modification times for an update (By default,
44     \t compares source file's creation time also).
45     """
46    
47    
48    
49     class PyRobocopier:
50     """ An advanced directory synchronization, updation
51     and file copying class """
52    
53     prog_name = "pyrobocopy.py"
54    
55     def __init__(self):
56    
57     self.__dir1 = ''
58     self.__dir2 = ''
59     self.__dcmp = None
60    
61     self.__copyfiles = True
62     self.__forcecopy = False
63     self.__copydirection = 0
64     self.__updatefiles = True
65     self.__creatdirs = True
66     self.__purge =False
67     self.__maketarget =False
68     self.__modtimeonly =False
69     self.__mainfunc = None
70    
71     # stat vars
72     self.__numdirs =0
73     self.__numfiles =0
74     self.__numdelfiles =0
75     self.__numdeldirs =0
76     self.__numnewdirs =0
77     self.__numupdates =0
78     self.__starttime = 0.0
79     self.__endtime = 0.0
80    
81     # failure stat vars
82     self.__numcopyfld =0
83     self.__numupdsfld =0
84     self.__numdirsfld =0
85     self.__numdelffld =0
86     self.__numdeldfld =0
87    
88     def parse_args(self, arguments):
89     """ Parse arguments """
90    
91     import getopt
92    
93     shortargs = "supncm"
94     longargs = ["synchronize=", "update=", "purge=", "nodirection=", "create=", "modtime="]
95    
96     try:
97     optlist, args = getopt.getopt( arguments, shortargs, longargs )
98     except getopt.GetoptError, e:
99     print e
100     return None
101    
102     allargs = []
103     if len(optlist):
104     allargs = [x[0] for x in optlist]
105    
106     allargs.extend( args )
107     self.__setargs( allargs )
108    
109     def __setargs(self, argslist):
110     """ Sets internal variables using arguments """
111    
112     for option in argslist:
113     if option.lower() in ('-s', '--synchronize'):
114     self.__mainfunc = self.synchronize
115     elif option.lower() in ('-u', '--update'):
116     self.__mainfunc = self.update
117     elif option.lower() in ('-d', '--diff'):
118     self.__mainfunc = self.dirdiff
119     elif option.lower() in ('-p', '--purge'):
120     self.__purge = True
121     elif option.lower() in ('-n', '--nodirection'):
122     self.__copydirection = 2
123     elif option.lower() in ('-f', '--force'):
124     self.__forcecopy = True
125     elif option.lower() in ('-c', '--create'):
126     self.__maketarget = True
127     elif option.lower() in ('-m', '--modtime'):
128     self.__modtimeonly = True
129     else:
130     if self.__dir1=='':
131     self.__dir1 = option
132     elif self.__dir2=='':
133     self.__dir2 = option
134    
135     if self.__dir1=='' or self.__dir2=='':
136     sys.exit("Argument Error: Directory arguments not given!")
137     if not os.path.isdir(self.__dir1):
138     sys.exit("Argument Error: Source directory does not exist!")
139     if not self.__maketarget and not os.path.isdir(self.__dir2):
140     sys.exit("Argument Error: Target directory %s does not exist! (Try the -c option)." % self.__dir2)
141     if self.__mainfunc is None:
142     sys.exit("Argument Error: Specify an action (Diff, Synchronize or Update) ")
143    
144     self.__dcmp = filecmp.dircmp(self.__dir1, self.__dir2)
145    
146     def do_work(self):
147     """ Do work """
148    
149     self.__starttime = time.time()
150    
151     if not os.path.isdir(self.__dir2):
152     if self.__maketarget:
153     print 'Creating directory', self.__dir2
154     try:
155     os.makedirs(self.__dir2)
156     except Exception, e:
157     print e
158     return None
159    
160     # All right!
161     self.__mainfunc()
162     self.__endtime = time.time()
163    
164     def __dowork(self, dir1, dir2, copyfunc = None, updatefunc = None):
165     """ Private attribute for doing work """
166    
167     print 'Source directory: ', dir1, ':'
168    
169     self.__numdirs += 1
170     self.__dcmp = filecmp.dircmp(dir1, dir2)
171    
172     # Files & directories only in target directory
173     if self.__purge:
174     for f2 in self.__dcmp.right_only:
175     fullf2 = os.path.join(dir2, f2)
176     print 'Deleting ',fullf2
177     try:
178     if os.path.isfile(fullf2):
179    
180     try:
181     os.remove(fullf2)
182     self.__numdelfiles += 1
183     except OSError, e:
184     print e
185     self.__numdelffld += 1
186     elif os.path.isdir(fullf2):
187     try:
188     shutil.rmtree( fullf2, True )
189     self.__numdeldirs += 1
190     except shutil.Error, e:
191     print e
192     self.__numdeldfld += 1
193    
194     except Exception, e: # of any use ?
195     print e
196     continue
197    
198    
199     # Files & directories only in source directory
200     for f1 in self.__dcmp.left_only:
201     try:
202     st = os.stat(os.path.join(dir1, f1))
203     except os.error:
204     continue
205    
206     if stat.S_ISREG(st.st_mode):
207     if copyfunc: copyfunc(f1, dir1, dir2)
208     elif stat.S_ISDIR(st.st_mode):
209     fulld1 = os.path.join(dir1, f1)
210     fulld2 = os.path.join(dir2, f1)
211    
212     if self.__creatdirs:
213     try:
214     # Copy tree
215     print 'Copying tree', fulld2
216     shutil.copytree(fulld1, fulld2)
217     self.__numnewdirs += 1
218     print 'Done.'
219     except shutil.Error, e:
220     print e
221     self.__numdirsfld += 1
222    
223     # jump to next file/dir in loop since this op failed
224     continue
225    
226     # Call tail recursive
227     # if os.path.exists(fulld2):
228     # self.__dowork(fulld1, fulld2, copyfunc, updatefunc)
229    
230     # common files/directories
231     for f1 in self.__dcmp.common:
232     try:
233     st = os.stat(os.path.join(dir1, f1))
234     except os.error:
235     continue
236    
237     if stat.S_ISREG(st.st_mode):
238     if updatefunc: updatefunc(f1, dir1, dir2)
239     elif stat.S_ISDIR(st.st_mode):
240     fulld1 = os.path.join(dir1, f1)
241     fulld2 = os.path.join(dir2, f1)
242     # Call tail recursive
243     self.__dowork(fulld1, fulld2, copyfunc, updatefunc)
244    
245    
246     def __copy(self, filename, dir1, dir2):
247     """ Private function for copying a file """
248    
249     # NOTE: dir1 is source & dir2 is target
250     if self.__copyfiles:
251    
252     print 'Copying file', filename, dir1, dir2
253     try:
254     if self.__copydirection== 0 or self.__copydirection == 2: # source to target
255    
256     if not os.path.exists(dir2):
257     if self.__forcecopy:
258     os.chmod(os.path.dirname(dir2), 0777)
259     try:
260     os.makedirs(dir1)
261     except OSError, e:
262     print e
263     self.__numdirsfld += 1
264    
265     if self.__forcecopy:
266     os.chmod(dir2, 0777)
267    
268     sourcefile = os.path.join(dir1, filename)
269     try:
270     shutil.copy(sourcefile, dir2)
271     self.__numfiles += 1
272     except (IOError, OSError), e:
273     print e
274     self.__numcopyfld += 1
275    
276     elif self.__copydirection==1 or self.__copydirection == 2: # target to source
277    
278     if not os.path.exists(dir1):
279     if self.__forcecopy:
280     os.chmod(os.path.dirname(dir1), 0777)
281    
282     try:
283     os.makedirs(dir1)
284     except OSError, e:
285     print e
286     self.__numdirsfld += 1
287    
288     targetfile = os.path.abspath(os.path.join(dir1, filename))
289     if self.__forcecopy:
290     os.chmod(dir1, 0777)
291    
292     sourcefile = os.path.join(dir2, filename)
293    
294     try:
295     shutil.copy(sourcefile, dir1)
296     self.__numfiles += 1
297     except (IOError, OSError), e:
298     print e
299     self.__numcopyfld += 1
300    
301     except Exception, e:
302     print 'Error copying file', filename, e
303    
304     def __cmptimestamps(self, filest1, filest2):
305     """ Compare time stamps of two files and return True
306     if file1 (source) is more recent than file2 (target) """
307    
308     return ((filest1.st_mtime > filest2.st_mtime) or \
309     (not self.__modtimeonly and (filest1.st_ctime > filest2.st_mtime)))
310    
311     def __update(self, filename, dir1, dir2):
312     """ Private function for updating a file based on
313     last time stamp of modification """
314    
315     print 'Updating file', filename
316    
317     # NOTE: dir1 is source & dir2 is target
318     if self.__updatefiles:
319    
320     file1 = os.path.join(dir1, filename)
321     file2 = os.path.join(dir2, filename)
322    
323     try:
324     st1 = os.stat(file1)
325     st2 = os.stat(file2)
326     except os.error:
327     return -1
328    
329     # Update will update in both directions depending
330     # on the timestamp of the file & copy-direction.
331    
332     if self.__copydirection==0 or self.__copydirection == 2:
333    
334     # Update file if file's modification time is older than
335     # source file's modification time, or creation time. Sometimes
336     # it so happens that a file's creation time is newer than it's
337     # modification time! (Seen this on windows)
338     if self.__cmptimestamps( st1, st2 ):
339     print 'Updating file ', file2 # source to target
340     try:
341     if self.__forcecopy:
342     os.chmod(file2, 0666)
343    
344     try:
345     shutil.copy(file1, file2)
346     self.__numupdates += 1
347     return 0
348     except (IOError, OSError), e:
349     print e
350     self.__numupdsfld += 1
351     return -1
352    
353     except Exception, e:
354     print e
355     return -1
356    
357     elif self.__copydirection==1 or self.__copydirection == 2:
358    
359     # Update file if file's modification time is older than
360     # source file's modification time, or creation time. Sometimes
361     # it so happens that a file's creation time is newer than it's
362     # modification time! (Seen this on windows)
363     if self.__cmptimestamps( st2, st1 ):
364     print 'Updating file ', file1 # target to source
365     try:
366     if self.__forcecopy:
367     os.chmod(file1, 0666)
368    
369     try:
370     shutil.copy(file2, file1)
371     self.__numupdates += 1
372     return 0
373     except (IOError, OSError), e:
374     print e
375     self.__numupdsfld += 1
376     return -1
377    
378     except Exception, e:
379     print e
380     return -1
381    
382     return -1
383    
384     def __dirdiffandcopy(self, dir1, dir2):
385     """ Private function which does directory diff & copy """
386     self.__dowork(dir1, dir2, self.__copy)
387    
388     def __dirdiffandupdate(self, dir1, dir2):
389     """ Private function which does directory diff & update """
390     self.__dowork(dir1, dir2, None, self.__update)
391    
392     def __dirdiffcopyandupdate(self, dir1, dir2):
393     """ Private function which does directory diff, copy and update (synchro) """
394     self.__dowork(dir1, dir2, self.__copy, self.__update)
395    
396     def __dirdiff(self):
397     """ Private function which only does directory diff """
398    
399     if self.__dcmp.left_only:
400     print 'Only in', self.__dir1
401     for x in self.__dcmp.left_only:
402     print '>>', x
403    
404     if self.__dcmp.right_only:
405     print 'Only in', self.__dir2
406     for x in self.__dcmp.right_only:
407     print '<<', x
408    
409     if self.__dcmp.common:
410     print 'Common to', self.__dir1,' and ',self.__dir2
411     print
412     for x in self.__dcmp.common:
413     print '--', x
414     else:
415     print 'No common files or sub-directories!'
416    
417     def synchronize(self):
418     """ Synchronize will try to synchronize two directories w.r.t
419     each other's contents, copying files if necessary from source
420     to target, and creating directories if necessary. If the optional
421     argument purge is True, directories in target (dir2) that are
422     not present in the source (dir1) will be deleted . Synchronization
423     is done in the direction of source to target """
424    
425     self.__copyfiles = True
426     self.__updatefiles = True
427     self.__creatdirs = True
428     self.__copydirection = 0
429    
430     print 'Synchronizing directory', self.__dir2, 'with', self.__dir1 ,'\n'
431     self.__dirdiffcopyandupdate(self.__dir1, self.__dir2)
432    
433     def update(self):
434     """ Update will try to update the target directory
435     w.r.t source directory. Only files that are common
436     to both directories will be updated, no new files
437     or directories are created """
438    
439     self.__copyfiles = False
440     self.__updatefiles = True
441     self.__purge = False
442     self.__creatdirs = False
443    
444     print 'Updating directory', self.__dir2, 'from', self.__dir1 , '\n'
445     self.__dirdiffandupdate(self.__dir1, self.__dir2)
446    
447     def dirdiff(self):
448     """ Only report difference in content between two
449     directories """
450    
451     self.__copyfiles = False
452     self.__updatefiles = False
453     self.__purge = False
454     self.__creatdirs = False
455     self.__updatefiles = False
456    
457     print 'Difference of directory ', self.__dir2, 'from', self.__dir1 , '\n'
458     self.__dirdiff()
459    
460     def report(self):
461     """ Print report of work at the end """
462    
463     # We need only the first 4 significant digits
464     tt = (str(self.__endtime - self.__starttime))[:4]
465    
466     print '\nPython robocopier finished in',tt, 'seconds.'
467     print self.__numdirs, 'directories parsed,',self.__numfiles, 'files copied.'
468     if self.__numdelfiles:
469     print self.__numdelfiles, 'files were purged.'
470     if self.__numdeldirs:
471     print self.__numdeldirs, 'directories were purged.'
472     if self.__numnewdirs:
473     print self.__numnewdirs, 'directories were created.'
474     if self.__numupdates:
475     print self.__numupdates, 'files were updated by timestamp.'
476    
477     # Failure stats
478     print '\n'
479     if self.__numcopyfld:
480     print self.__numcopyfld, 'files could not be copied.'
481     if self.__numdirsfld:
482     print self.__numdirsfld, 'directories could not be created.'
483     if self.__numupdsfld:
484     print self.__numupdsfld, 'files could not be updated.'
485     if self.__numdeldfld:
486     print self.__numdeldfld, 'directories could not be purged.'
487     if self.__numdelffld:
488     print self.__numdelffld, 'files could not be purged.'
489    
490     if __name__=="__main__":
491     import sys
492    
493     if len(sys.argv)<2:
494     sys.exit( usage() % PyRobocopier.prog_name )
495    
496     copier = PyRobocopier()
497     copier.parse_args(sys.argv[1:])
498     copier.do_work()
499    
500     # print report at the end
501     copier.report()