NinoCode/Active_prgs/Gather_ImgDir_Sizes.py

"""

Gather_ImgDir_Sizes

This program will be the second part of a system that gathers the dir sizes of the image clm folder.
It requires a text file of all possible C_D dirs and a text file of the client matters that you want it
to find.


NOTES:  The original program did the gathering en mass. Now that we've gathered all of the sizes of the 
        old and existing database, we now want a program that will just give a total of what X CLM is on 
        the dis and only from dis18, dis19 and dis20. It will still take a txt file of clms as input but
        now will give you the totals per CLM.  Will probably put a GUI on this eventually too. Would also
        be nice if it created a spreadsheet as output where it did calculations of sizes in cost, etc. It
        could also gather past sizes to give you stats on increases.

UPDATES:
    - 20110609 Updated this program to once again point to a list of servers, instead of hard coded
                per request of team. They wanted an easier way to edit the list that this program
                scans.
    - 20110609 Updated so that this one program can run in the 'old' way, (Verbose), and the new way
                (non-verbose).  The shortcut controls how it outputs.
    - 20110609 Updated to support floating point sizes, so it gathers true sizes.

"       The program was upgraded to now use floating point math, so the sizes will be much more specific.
"       The list of 'shares' that it scans is now in a text file called C_D_MasterList.txt, per each of your instances.  You each have a separate folder, so you can feel free to add, remove and update this file.  If a new share is added, you can just add it to this file.
"       I also rewrote the crawler so that it also supports UTF-8, meaning that it should find less errors and will be a bit more robust in it's size gatherings.
"       I added a routine that also puts a total at the end of each matter, even if you run the verbose version, so you should not have to sum these manually anymore.
"       Finally, the program can run in both verbose and non-verbose mode.  I created two shortcuts to make it easier to control this. Just double click.
    
    
"""

import os, directorySize2, time, sys


if __name__ == '__main__':
    try:
        processMethod = sys.argv[1]
        if processMethod == '-SMALL':
            processMethod = None
        
    except:
        processMethod = None
    baseDir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))[0]
    mattersToProcessList = open(os.path.join(baseDir,'ClientMatterList.txt')).readlines()
    imgLocationsList = open(os.path.join(baseDir,'C_D_MasterList.txt')).readlines()
    outputFile = open(os.path.join(baseDir,"%s_Client_Sizes.txt"%(time.strftime('%m-%d-%Y'))),'w')

    
    #mattersToProcessList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Aaron\App\ClientMatterList.txt").readlines()
    #mattersToProcessList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Dave\App\ClientMatterList.txt").readlines()
    ##mattersToProcessList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Manny\WIP\dave\ClientMatterList.txt").readlines()
    #imgLocationsList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Manny\APP\C_D_MasterList.txt").readlines()
##    imgLocationsList = [r'\\lisdisdss01\dis18\c_d',r'\\lisdisdss01\dis19\c_d',r'\\lisdisdss01\dis20\c_d',
##                        r'\\lisdisdss01\dis01\c_d',r'\\lisdisdss01\dis02\c_d',r'\\lisdisdss01\dis03\c_d',
##                        r'\\lisdisdss01\dis04\c_d',r'\\lisdisdss01\dis05\c_d',r'\\lisdisdss01\dis06\c_d',
##                        r'\\lisdisdss01\dis07\c_d',r'\\lisdisdss01\dis08\c_d',r'\\lisdisdss01\dis01\windows\C_D',
##                        r'\\lisdisdss01\dis09\c_d',r'\\lisdisdss01\dis10\c_d',r'\\lisdisdss01\dis11\c_d',
##                        r'\\lisdisdss01\dis12\c_d',r'\\lisdisdss01\dis13\c_d',r'\\lisdisdss01\dis14\c_d',
##                        r'\\lisdisdss01\dis15\c_d',r'\\lisdisdss01\dis16\c_d',r'\\lisdisdss01\dis17\c_d',
##                        r'\\lisdisdss01\dis21\c_d', r'\\lisdisdss01\dis22\c_d',r'\\lisdisdss01\dis23\c_d',
##                        r'\\lisdisdss01\dis24\c_d',r'\\lisdisdss01\dis25\c_d',r'\\lisdisdss01\dis26\c_d']
    #outputFile = open(r"\\lisdisdss01\ercdis12\Admin\Share\Aaron\App\%s_Client_Sizes.txt"%(time.strftime('%m-%d-%Y')),'w')
    imageDirCLMDict = {}
    print "\n\n"
    print "-"*80
    print "Gather Image Directory Sizes version 2.0\nCreated by Emanuel Borges\n"
    if processMethod:
        print "Verbose ON"
    else:
        print "Verbose OFF"
    print "-"*80
    print "\n\n"
    ## Make the master dictionary of clientmatternumbers = list of dirs
    print "Creating the master dictionary..."
    for loc in imgLocationsList:
        loc = loc.replace("\n","")
        dirList = os.listdir(loc)
        for x in dirList:
            try:
                if imageDirCLMDict[x]:
                    imageDirCLMDict[x].append(os.path.join(loc, x))
            except:
                imageDirCLMDict[x] = [os.path.join(loc, x)]
    print "Dictionary complete.\n"

    print "Processing Matter List ...\n\n"
    for matter in mattersToProcessList:
        matterExists = True
        matter = matter.replace("\n","")
        print "Processing %s..."% matter
        outputFile.write(matter + "|")
        try:
            dataDirs = imageDirCLMDict[matter]
        except:
            print "%s does not exist on the image servers!"% matter
            outputFile.write("Does not exist on image server\n\n")
            matterExists = False
        if matterExists:
            numberOfDataDirs = len(dataDirs)
            if processMethod:
                outputFile.write("There are %d data directories for this matter.\n"% numberOfDataDirs)
            sizes = []
            for dataSite in dataDirs:
                print "parsing %s..."% dataSite
                if processMethod:
                    outputFile.write(matter + "|"+dataSite + "|"  )
                try:
                    dataDirSize = directorySize2.get_dir_size(dataSite,True)
                    if processMethod:
                        outputFile.write(directorySize2.pretty_filesize2(dataDirSize) + "\n")
                    sizes.append(dataDirSize)
                except:
                    outputFile.write("<ERROR!!>\n")
                    print matter + " was zero.\n"
            print matter + " total is:"
            totalSize = 0
            for size in sizes:
                totalSize += size
            finalSize = directorySize2.pretty_filesize2(totalSize)
            if processMethod:
                outputFile.write('TOTAL = ' + finalSize + "\n")
            else:
                outputFile.write(finalSize + "\n")
            print finalSize
            print "\n"
            #outputFile.write("|")
            #for s in sizes:
            #    outputFile.writelines(s+",")
            outputFile.write("\n\n")
    outputFile.write("Processing Complete.\n")    
    outputFile.close()
    s = raw_input("Processing Complete!\nAll sizes have been written to log file.\n\nPress Enter key to exit.")

            
Revision:	8
Committed:	Sat May 5 04:21:19 2012 UTC (13 years, 10 months ago) by ninoborges
Content type:	text/x-python
File size:	7230 byte(s)
Log Message:	Initial Import
#	User	Rev	Content
1	ninoborges	8	"""
2
3			Gather_ImgDir_Sizes
4
5			This program will be the second part of a system that gathers the dir sizes of the image clm folder.
6			It requires a text file of all possible C_D dirs and a text file of the client matters that you want it
7			to find.
8
9
10
11			NOTES: The original program did the gathering en mass. Now that we've gathered all of the sizes of the
12			old and existing database, we now want a program that will just give a total of what X CLM is on
13			the dis and only from dis18, dis19 and dis20. It will still take a txt file of clms as input but
14			now will give you the totals per CLM. Will probably put a GUI on this eventually too. Would also
15			be nice if it created a spreadsheet as output where it did calculations of sizes in cost, etc. It
16			could also gather past sizes to give you stats on increases.
17
18			UPDATES:
19			- 20110609 Updated this program to once again point to a list of servers, instead of hard coded
20			per request of team. They wanted an easier way to edit the list that this program
21			scans.
22			- 20110609 Updated so that this one program can run in the 'old' way, (Verbose), and the new way
23			(non-verbose). The shortcut controls how it outputs.
24			- 20110609 Updated to support floating point sizes, so it gathers true sizes.
25
26			" The program was upgraded to now use floating point math, so the sizes will be much more specific.
27			" The list of 'shares' that it scans is now in a text file called C_D_MasterList.txt, per each of your instances. You each have a separate folder, so you can feel free to add, remove and update this file. If a new share is added, you can just add it to this file.
28			" I also rewrote the crawler so that it also supports UTF-8, meaning that it should find less errors and will be a bit more robust in it's size gatherings.
29			" I added a routine that also puts a total at the end of each matter, even if you run the verbose version, so you should not have to sum these manually anymore.
30			" Finally, the program can run in both verbose and non-verbose mode. I created two shortcuts to make it easier to control this. Just double click.
31
32
33
34
35			"""
36
37			import os, directorySize2, time, sys
38
39
40
41
42			if __name__ == '__main__':
43			try:
44			processMethod = sys.argv[1]
45			if processMethod == '-SMALL':
46			processMethod = None
47
48			except:
49			processMethod = None
50			baseDir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))[0]
51			mattersToProcessList = open(os.path.join(baseDir,'ClientMatterList.txt')).readlines()
52			imgLocationsList = open(os.path.join(baseDir,'C_D_MasterList.txt')).readlines()
53			outputFile = open(os.path.join(baseDir,"%s_Client_Sizes.txt"%(time.strftime('%m-%d-%Y'))),'w')
54
55
56			#mattersToProcessList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Aaron\App\ClientMatterList.txt").readlines()
57			#mattersToProcessList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Dave\App\ClientMatterList.txt").readlines()
58			##mattersToProcessList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Manny\WIP\dave\ClientMatterList.txt").readlines()
59			#imgLocationsList = open(r"\\lisdisdss01\ercdis12\Admin\Share\Manny\APP\C_D_MasterList.txt").readlines()
60			## imgLocationsList = [r'\\lisdisdss01\dis18\c_d',r'\\lisdisdss01\dis19\c_d',r'\\lisdisdss01\dis20\c_d',
61			## r'\\lisdisdss01\dis01\c_d',r'\\lisdisdss01\dis02\c_d',r'\\lisdisdss01\dis03\c_d',
62			## r'\\lisdisdss01\dis04\c_d',r'\\lisdisdss01\dis05\c_d',r'\\lisdisdss01\dis06\c_d',
63			## r'\\lisdisdss01\dis07\c_d',r'\\lisdisdss01\dis08\c_d',r'\\lisdisdss01\dis01\windows\C_D',
64			## r'\\lisdisdss01\dis09\c_d',r'\\lisdisdss01\dis10\c_d',r'\\lisdisdss01\dis11\c_d',
65			## r'\\lisdisdss01\dis12\c_d',r'\\lisdisdss01\dis13\c_d',r'\\lisdisdss01\dis14\c_d',
66			## r'\\lisdisdss01\dis15\c_d',r'\\lisdisdss01\dis16\c_d',r'\\lisdisdss01\dis17\c_d',
67			## r'\\lisdisdss01\dis21\c_d', r'\\lisdisdss01\dis22\c_d',r'\\lisdisdss01\dis23\c_d',
68			## r'\\lisdisdss01\dis24\c_d',r'\\lisdisdss01\dis25\c_d',r'\\lisdisdss01\dis26\c_d']
69			#outputFile = open(r"\\lisdisdss01\ercdis12\Admin\Share\Aaron\App\%s_Client_Sizes.txt"%(time.strftime('%m-%d-%Y')),'w')
70			imageDirCLMDict = {}
71			print "\n\n"
72			print "-"*80
73			print "Gather Image Directory Sizes version 2.0\nCreated by Emanuel Borges\n"
74			if processMethod:
75			print "Verbose ON"
76			else:
77			print "Verbose OFF"
78			print "-"*80
79			print "\n\n"
80			## Make the master dictionary of clientmatternumbers = list of dirs
81			print "Creating the master dictionary..."
82			for loc in imgLocationsList:
83			loc = loc.replace("\n","")
84			dirList = os.listdir(loc)
85			for x in dirList:
86			try:
87			if imageDirCLMDict[x]:
88			imageDirCLMDict[x].append(os.path.join(loc, x))
89			except:
90			imageDirCLMDict[x] = [os.path.join(loc, x)]
91			print "Dictionary complete.\n"
92
93			print "Processing Matter List ...\n\n"
94			for matter in mattersToProcessList:
95			matterExists = True
96			matter = matter.replace("\n","")
97			print "Processing %s..."% matter
98			outputFile.write(matter + "\|")
99			try:
100			dataDirs = imageDirCLMDict[matter]
101			except:
102			print "%s does not exist on the image servers!"% matter
103			outputFile.write("Does not exist on image server\n\n")
104			matterExists = False
105			if matterExists:
106			numberOfDataDirs = len(dataDirs)
107			if processMethod:
108			outputFile.write("There are %d data directories for this matter.\n"% numberOfDataDirs)
109			sizes = []
110			for dataSite in dataDirs:
111			print "parsing %s..."% dataSite
112			if processMethod:
113			outputFile.write(matter + "\|"+dataSite + "\|" )
114			try:
115			dataDirSize = directorySize2.get_dir_size(dataSite,True)
116			if processMethod:
117			outputFile.write(directorySize2.pretty_filesize2(dataDirSize) + "\n")
118			sizes.append(dataDirSize)
119			except:
120			outputFile.write("<ERROR!!>\n")
121			print matter + " was zero.\n"
122			print matter + " total is:"
123			totalSize = 0
124			for size in sizes:
125			totalSize += size
126			finalSize = directorySize2.pretty_filesize2(totalSize)
127			if processMethod:
128			outputFile.write('TOTAL = ' + finalSize + "\n")
129			else:
130			outputFile.write(finalSize + "\n")
131			print finalSize
132			print "\n"
133			#outputFile.write("\|")
134			#for s in sizes:
135			# outputFile.writelines(s+",")
136			outputFile.write("\n\n")
137			outputFile.write("Processing Complete.\n")
138			outputFile.close()
139			s = raw_input("Processing Complete!\nAll sizes have been written to log file.\n\nPress Enter key to exit.")
140
141