Active_prgs/MCP/MCP_CopyUp_Request.py

"""


MCP_CopyUp_Request.py

Created by
Emanuel Borges  
08.14.2010

TODO:  Next make it gather the tar size and update your sheet on what's been uploaded. Also make it
automatically make the destination point based on the case you select and the date.

NOTE:  What if the ftp connection times out while it's being compressed...?

"""

#import win32api
import os, tarfile, time, ConcordanceHelperTools, NinoGenTools, shutil,zipfile,MCP_Lib
from directorySize2 import pretty_filesize2


def ReportAndStore(caseName, totalSize, reviewPlatform):
    reportSize = pretty_filesize2(totalSize)
    epoch = str(int(time.time()))
    #casesDir = r"C:\Documents and Settings\eborges\My Documents\My Dropbox\Documents\Cases"
    casesList,officeCases, allCases, casesDir = MCP_Lib.GetCaseList()
    if caseName in casesList:
        if casesDir:
            casePath = os.path.join(casesDir,caseName)
            outputFile = open(os.path.join(casePath,"UploadReport.txt"),'a')
            
            outputFile.write(epoch + " | " + reviewPlatform + " | " + time.strftime('%Y%m%d') + " | " +reportSize + "\n")
            outputFile.close()
    print 'MCP: Connecting to the Access Database...'
    clientMatter = caseName.split("_(")[1]
    clientMatter = clientMatter[:-1]
    clientMatter = clientMatter.replace('-','.')
    #accessDB = MCP_Lib.AccessDBConnection(r"W:\Manny\testing\TPM_CaseTracking.mdb")
    accessDB = MCP_Lib.AccessDBConnection(r"\\chiads01\app\DS_CaseTrack\TPM_CaseTracking.mdb")
    accessDB.UpdateCaseUpload(clientMatter,reviewPlatform,{epoch:(time.strftime('%m/%d/%Y'),reportSize)})
    accessDB.CloseAccessConnection()
    print 'MCP: Access database updated.'

def CompressSelected(pathToCompress, workDir, fileName):
    ## check available space. Acutally this could take too long to calculate the source media.
    ## warn them to check the available size, for now.
    #availSize = win32api.GetDiskFreeSpaceEx(rootPath)
    ## create the tar
    if os.path.splitext(fileName)[1].upper() in [".ZIP", ".RAR",".TAR",".7Z"]:
        print "MCP: It's already compressed. Skipping step."
        tFileName = pathToCompress
    else:
        #print "this is second workdir %s"% workDir
        tFileName = os.path.join(workDir , fileName) +".tar"
        tFile = tarfile.open(tFileName,'w')
        tFile.add(pathToCompress,arcname = fileName,recursive = True)
        tFile.close()
    return tFileName
    
def CopyUpRemote(compressedFile,newLocation,expeObj):
    newTargetPath = os.path.split(newLocation)[0]
    pathExists = expeObj.TestPath(newTargetPath)
    tempPath = newTargetPath
    pathRemainingList = []
    while pathExists == False:
        tempPath, tempPart = os.path.split(tempPath)
        pathRemainingList.append(tempPart)
        pathExists = expeObj.TestPath(tempPath)
    if pathRemainingList:
        pathRemainingList.reverse()
        for pathPiece in pathRemainingList:
            tempPath = os.path.join(tempPath,pathPiece)
            expeObj.CreateNewPath(tempPath)
        
        
    drive,null = os.path.splitdrive(compressedFile)
    if drive.upper() == "C:":
        print "MCP: moving up %s"%compressedFile
        print "MCP: to %s"% newLocation
        errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
        if errCode == True:
            os.remove(compressedFile)
    else:
        print "MCP: copying up %s"%compressedFile
        print "MCP: to %s"% newLocation
        errCode = expeObj.CopyToLN(compressedFile,newTargetPath)

def CopyUpMWESFTP(compressedFile,newLocation,mweSftpObj):
    newTargetPath = os.path.split(newLocation)[0]
    pathExists = mweSftpObj.TestPath(newTargetPath)
    if pathExists:
        pass
    else:
        mweSftpObj.CreateNewPath(newTargetPath)
        
    drive,null = os.path.splitdrive(compressedFile)
    if drive.upper() == "C:":
        print "MCP: moving up %s"%compressedFile
        print "MCP: to %s"% newLocation
        mweSftpObj.CopyToDis(compressedFile,newTargetPath)
        os.remove(compressedFile)
    else:
        print "MCP: copying up %s"%compressedFile
        print "MCP: to %s"% newLocation
        mweSftpObj.CopyToDis(compressedFile,newTargetPath)

def CopyUp(compressedFile, newLocation):
    ##  WARNING! needs to be changes so it increments a dir. What if this dir already exists.
    if os.path.exists(os.path.split(newLocation)[0]):
        pass
    else:
        os.makedirs(os.path.split(newLocation)[0])
    
    ## if the pre archive is on C, move it. Else just copy it.ie a dvd with a zip on it.
    drive,null = os.path.splitdrive(compressedFile)
    if drive.upper() == "C:":
        print "MCP: copying up %s"%compressedFile
        print "MCP: to %s"% newLocation
        os.rename(compressedFile, newLocation)
    else:
        shutil.copy2(compressedFile, newLocation)
        print "MCP: moving up %s"%compressedFile
        print "MCP: to %s"% newLocation
    

def Process(caseName, workDir, startDir, platform = 'Concordance DIS', sourceMediaLocation = 'local'):
    ##This should be external too.
    availableShare = "Dis29"
    expeObj = None
    mweSftpObj = None
    if sourceMediaLocation == 'local':
        if platform == 'Concordance DIS':
            mweSftpObj = MCP_Lib.MweFtpConnection()
            connectionTest = os.path.isdir(os.path.join(r'\\lisdisdss01',availableShare))
        else:
            expeObj = MCP_Lib.ExpeDatConnection(platform)
            connectionTest = True
        if connectionTest:
            totalSize = NinoGenTools.Counter()
            networkDir = GetCopyToLocation(caseName, availableShare,expeObj,platform)
            if os.path.isfile(startDir):
                startDir,indFile = os.path.split(startDir)
                compressList = [indFile]
            else:
                compressList = os.listdir(startDir)
            for item in compressList:
                if item.upper() == 'SYSTEM VOLUME INFORMATION':
                    ## If it's the system folder skip it.
                    ##  Consider changing this to a try or to a list of stuff to skip.
                    pass
                else:
                    pathToCompress = os.path.join(startDir,item)
                    fileName = item
                    ## TODO: check to see if the file already exists. If so, skip that request and give them a dialog.
                    print "MCP: Compressing %s"% item
                    compressedFile = CompressSelected(pathToCompress, workDir, fileName)
                    #print compressedFile
                    print "MCP: Done!"
                    print "MCP: Gathering size..."
                    if os.path.splitext(fileName)[1].upper() in [".ZIP"]:
                        print "MCP: It's a ZIP, so gathering uncompressed size..."
                        try:
                            tempZipSize = 0
                            tempZip = zipfile.ZipFile(compressedFile,'r')
                            for zipInfo in tempZip.infolist():
                                tempZipSize = tempZipSize + zipInfo.file_size
                            tempZip.close()
                            totalSize.inc(tempZipSize)
                            print "MCP: Uncompressed size gathered."
                        except:
                            print "MCP: Warning: Zip error. Defaulting to compressed size."
                            totalSize.inc(os.path.getsize(compressedFile)) 
                    else:
                        totalSize.inc(os.path.getsize(compressedFile))
                        print "MCP: Size gathered."
                    print "MCP: Copying up %s"% item
                    #print "comressedFile is %s"% compressedFile
                    ##  Copy or move it up.
                    #CopyUp(compressedFile, os.path.join(networkDir+"\\"+time.strftime('%Y%m%d'), os.path.split(compressedFile)[1]))
                    if platform == 'Concordance DIS':
                        ## SFTP coping is now only way.
                        CopyUpMWESFTP(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),mweSftpObj)
                        
                        ## Wan copying totally disabled.
                        ## CopyUp(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]))
                    else:
                        CopyUpRemote(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),expeObj)
                    print "MCP: Done!"
            print "MCP: Saving sizes..."
            ReportAndStore(caseName, totalSize.count, platform)
            print "MCP: Saving complete."
            print "MCP: All Items processed and copied!"
        else:
            print "MCP: ERROR!!! You are not connected to the DIS.\nPlease connect to the DIS and try your request again. \n\n\tEnd Of Line.\n"
    else:
        ## User is copying from LN to LN
        expeObj = MCP_Lib.ExpeDatConnection(platform)
        totalSize = expeObj.GatherSize(startDir)
        ReportAndStore(caseName, totalSize, platform)
        
        
    if mweSftpObj:
        mweSftpObj.CloseConnection()
    raw_input("\n\nMCP: Press Enter key to exit.\n\n\tEnd Of Line.\n")

def GetCopyToLocation(caseName, availableShare,expeObj,platform = 'Concordance DIS'):
    clientMatter = caseName.split("_(")[1]
    clientMatter = clientMatter[:-1]
    if platform == 'Concordance DIS':
        parsedPath = ConcordanceHelperTools.ParseClientMatter(clientMatter)
        parsedClientMatter = os.path.split(parsedPath)[1]
        copyToLocation = os.path.join(r"\\lisdisdss01\%s\C_D"%availableShare,parsedClientMatter)
        
        copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
        
        ## Check to see if this path already exists.  If it does, keep incrementing it until it dosent.
        dirIncrement = NinoGenTools.Counter(1)
        if os.path.exists(copyToLocation):
            copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
        while os.path.exists(copyToLocation):
            copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
            
        ## Now that it's SFTP only, remove the part before c_d
        copyToLocation = copyToLocation.split("\\\\lisdisdss01\\%s\\"%availableShare)[1]
    else:
        parsedPath = ConcordanceHelperTools.ParseClientMatterRemote(clientMatter)
        copyToLocation = parsedPath
        
        copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
        
        ## Check to see if this path already exists.  If it does, keep incrementing it until it dosent.
        dirIncrement = NinoGenTools.Counter(1)
        if expeObj.TestPath(copyToLocation) == True:
            copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
        while expeObj.TestPath(copyToLocation) == True:
            copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
            
    return copyToLocation

if __name__ == '__main__':
    caseName = "DaVita-Dallas_(039323-0308)"
    workDir = r"C:\Documents and Settings\eborges\Desktop\Working"
    #workDir = r"E:"
    startDir = r"C:\Documents and Settings\eborges\Desktop\Working\STCL_EML_IMAGE_1101.zip"
    Process(caseName, workDir, startDir)
    
    
Revision:	235
Committed:	Wed Dec 19 21:43:15 2012 UTC (13 years, 3 months ago) by nino.borges
Content type:	text/x-python
File size:	11662 byte(s)
Log Message:	MCP: Copy up split into source media local and remote.
#	User	Rev	Content
1	ninoborges	8	"""
2
3
4			MCP_CopyUp_Request.py
5
6			Created by
7			Emanuel Borges
8			08.14.2010
9
10			TODO: Next make it gather the tar size and update your sheet on what's been uploaded. Also make it
11			automatically make the destination point based on the case you select and the date.
12
13			NOTE: What if the ftp connection times out while it's being compressed...?
14
15			"""
16
17			#import win32api
18			import os, tarfile, time, ConcordanceHelperTools, NinoGenTools, shutil,zipfile,MCP_Lib
19			from directorySize2 import pretty_filesize2
20
21
22			def ReportAndStore(caseName, totalSize, reviewPlatform):
23			reportSize = pretty_filesize2(totalSize)
24			epoch = str(int(time.time()))
25			#casesDir = r"C:\Documents and Settings\eborges\My Documents\My Dropbox\Documents\Cases"
26	nino.borges	189	casesList,officeCases, allCases, casesDir = MCP_Lib.GetCaseList()
27			if caseName in casesList:
28			if casesDir:
29			casePath = os.path.join(casesDir,caseName)
30			outputFile = open(os.path.join(casePath,"UploadReport.txt"),'a')
31
32			outputFile.write(epoch + " \| " + reviewPlatform + " \| " + time.strftime('%Y%m%d') + " \| " +reportSize + "\n")
33			outputFile.close()
34	ninoborges	8	print 'MCP: Connecting to the Access Database...'
35			clientMatter = caseName.split("_(")[1]
36			clientMatter = clientMatter[:-1]
37			clientMatter = clientMatter.replace('-','.')
38			#accessDB = MCP_Lib.AccessDBConnection(r"W:\Manny\testing\TPM_CaseTracking.mdb")
39			accessDB = MCP_Lib.AccessDBConnection(r"\\chiads01\app\DS_CaseTrack\TPM_CaseTracking.mdb")
40			accessDB.UpdateCaseUpload(clientMatter,reviewPlatform,{epoch:(time.strftime('%m/%d/%Y'),reportSize)})
41			accessDB.CloseAccessConnection()
42			print 'MCP: Access database updated.'
43
44			def CompressSelected(pathToCompress, workDir, fileName):
45			## check available space. Acutally this could take too long to calculate the source media.
46			## warn them to check the available size, for now.
47			#availSize = win32api.GetDiskFreeSpaceEx(rootPath)
48			## create the tar
49			if os.path.splitext(fileName)[1].upper() in [".ZIP", ".RAR",".TAR",".7Z"]:
50			print "MCP: It's already compressed. Skipping step."
51			tFileName = pathToCompress
52			else:
53			#print "this is second workdir %s"% workDir
54			tFileName = os.path.join(workDir , fileName) +".tar"
55			tFile = tarfile.open(tFileName,'w')
56	nino.borges	151	tFile.add(pathToCompress,arcname = fileName,recursive = True)
57	ninoborges	8	tFile.close()
58			return tFileName
59
60			def CopyUpRemote(compressedFile,newLocation,expeObj):
61			newTargetPath = os.path.split(newLocation)[0]
62			pathExists = expeObj.TestPath(newTargetPath)
63			tempPath = newTargetPath
64			pathRemainingList = []
65			while pathExists == False:
66			tempPath, tempPart = os.path.split(tempPath)
67			pathRemainingList.append(tempPart)
68			pathExists = expeObj.TestPath(tempPath)
69			if pathRemainingList:
70			pathRemainingList.reverse()
71			for pathPiece in pathRemainingList:
72			tempPath = os.path.join(tempPath,pathPiece)
73			expeObj.CreateNewPath(tempPath)
74
75
76			drive,null = os.path.splitdrive(compressedFile)
77			if drive.upper() == "C:":
78			print "MCP: moving up %s"%compressedFile
79			print "MCP: to %s"% newLocation
80			errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
81			if errCode == True:
82			os.remove(compressedFile)
83			else:
84			print "MCP: copying up %s"%compressedFile
85			print "MCP: to %s"% newLocation
86			errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
87
88			def CopyUpMWESFTP(compressedFile,newLocation,mweSftpObj):
89			newTargetPath = os.path.split(newLocation)[0]
90			pathExists = mweSftpObj.TestPath(newTargetPath)
91			if pathExists:
92			pass
93			else:
94			mweSftpObj.CreateNewPath(newTargetPath)
95
96			drive,null = os.path.splitdrive(compressedFile)
97			if drive.upper() == "C:":
98			print "MCP: moving up %s"%compressedFile
99			print "MCP: to %s"% newLocation
100			mweSftpObj.CopyToDis(compressedFile,newTargetPath)
101			os.remove(compressedFile)
102			else:
103			print "MCP: copying up %s"%compressedFile
104			print "MCP: to %s"% newLocation
105			mweSftpObj.CopyToDis(compressedFile,newTargetPath)
106
107			def CopyUp(compressedFile, newLocation):
108			## WARNING! needs to be changes so it increments a dir. What if this dir already exists.
109			if os.path.exists(os.path.split(newLocation)[0]):
110			pass
111			else:
112			os.makedirs(os.path.split(newLocation)[0])
113
114			## if the pre archive is on C, move it. Else just copy it.ie a dvd with a zip on it.
115			drive,null = os.path.splitdrive(compressedFile)
116			if drive.upper() == "C:":
117			print "MCP: copying up %s"%compressedFile
118			print "MCP: to %s"% newLocation
119			os.rename(compressedFile, newLocation)
120			else:
121			shutil.copy2(compressedFile, newLocation)
122			print "MCP: moving up %s"%compressedFile
123			print "MCP: to %s"% newLocation
124
125
126	nino.borges	235	def Process(caseName, workDir, startDir, platform = 'Concordance DIS', sourceMediaLocation = 'local'):
127	ninoborges	8	##This should be external too.
128			availableShare = "Dis29"
129			expeObj = None
130			mweSftpObj = None
131	nino.borges	235	if sourceMediaLocation == 'local':
132			if platform == 'Concordance DIS':
133			mweSftpObj = MCP_Lib.MweFtpConnection()
134			connectionTest = os.path.isdir(os.path.join(r'\\lisdisdss01',availableShare))
135	ninoborges	8	else:
136	nino.borges	235	expeObj = MCP_Lib.ExpeDatConnection(platform)
137			connectionTest = True
138			if connectionTest:
139			totalSize = NinoGenTools.Counter()
140			networkDir = GetCopyToLocation(caseName, availableShare,expeObj,platform)
141			if os.path.isfile(startDir):
142			startDir,indFile = os.path.split(startDir)
143			compressList = [indFile]
144	ninoborges	8	else:
145	nino.borges	235	compressList = os.listdir(startDir)
146			for item in compressList:
147			if item.upper() == 'SYSTEM VOLUME INFORMATION':
148			## If it's the system folder skip it.
149			## Consider changing this to a try or to a list of stuff to skip.
150			pass
151	ninoborges	8	else:
152	nino.borges	235	pathToCompress = os.path.join(startDir,item)
153			fileName = item
154			## TODO: check to see if the file already exists. If so, skip that request and give them a dialog.
155			print "MCP: Compressing %s"% item
156			compressedFile = CompressSelected(pathToCompress, workDir, fileName)
157			#print compressedFile
158			print "MCP: Done!"
159			print "MCP: Gathering size..."
160			if os.path.splitext(fileName)[1].upper() in [".ZIP"]:
161			print "MCP: It's a ZIP, so gathering uncompressed size..."
162			try:
163			tempZipSize = 0
164			tempZip = zipfile.ZipFile(compressedFile,'r')
165			for zipInfo in tempZip.infolist():
166			tempZipSize = tempZipSize + zipInfo.file_size
167			tempZip.close()
168			totalSize.inc(tempZipSize)
169			print "MCP: Uncompressed size gathered."
170			except:
171			print "MCP: Warning: Zip error. Defaulting to compressed size."
172			totalSize.inc(os.path.getsize(compressedFile))
173			else:
174			totalSize.inc(os.path.getsize(compressedFile))
175			print "MCP: Size gathered."
176			print "MCP: Copying up %s"% item
177			#print "comressedFile is %s"% compressedFile
178			## Copy or move it up.
179			#CopyUp(compressedFile, os.path.join(networkDir+"\\"+time.strftime('%Y%m%d'), os.path.split(compressedFile)[1]))
180			if platform == 'Concordance DIS':
181			## SFTP coping is now only way.
182			CopyUpMWESFTP(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),mweSftpObj)
183
184			## Wan copying totally disabled.
185			## CopyUp(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]))
186			else:
187			CopyUpRemote(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),expeObj)
188			print "MCP: Done!"
189			print "MCP: Saving sizes..."
190			ReportAndStore(caseName, totalSize.count, platform)
191			print "MCP: Saving complete."
192			print "MCP: All Items processed and copied!"
193			else:
194			print "MCP: ERROR!!! You are not connected to the DIS.\nPlease connect to the DIS and try your request again. \n\n\tEnd Of Line.\n"
195	ninoborges	8	else:
196	nino.borges	235	## User is copying from LN to LN
197			expeObj = MCP_Lib.ExpeDatConnection(platform)
198			totalSize = expeObj.GatherSize(startDir)
199			ReportAndStore(caseName, totalSize, platform)
200
201
202	ninoborges	8	if mweSftpObj:
203			mweSftpObj.CloseConnection()
204			raw_input("\n\nMCP: Press Enter key to exit.\n\n\tEnd Of Line.\n")
205
206			def GetCopyToLocation(caseName, availableShare,expeObj,platform = 'Concordance DIS'):
207			clientMatter = caseName.split("_(")[1]
208			clientMatter = clientMatter[:-1]
209			if platform == 'Concordance DIS':
210			parsedPath = ConcordanceHelperTools.ParseClientMatter(clientMatter)
211			parsedClientMatter = os.path.split(parsedPath)[1]
212			copyToLocation = os.path.join(r"\\lisdisdss01\%s\C_D"%availableShare,parsedClientMatter)
213
214			copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
215
216			## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
217			dirIncrement = NinoGenTools.Counter(1)
218			if os.path.exists(copyToLocation):
219			copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
220			dirIncrement.inc()
221			while os.path.exists(copyToLocation):
222			copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
223			dirIncrement.inc()
224
225			## Now that it's SFTP only, remove the part before c_d
226			copyToLocation = copyToLocation.split("\\\\lisdisdss01\\%s\\"%availableShare)[1]
227			else:
228			parsedPath = ConcordanceHelperTools.ParseClientMatterRemote(clientMatter)
229			copyToLocation = parsedPath
230
231			copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
232
233			## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
234			dirIncrement = NinoGenTools.Counter(1)
235			if expeObj.TestPath(copyToLocation) == True:
236			copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
237			dirIncrement.inc()
238			while expeObj.TestPath(copyToLocation) == True:
239			copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
240			dirIncrement.inc()
241
242			return copyToLocation
243
244			if __name__ == '__main__':
245			caseName = "DaVita-Dallas_(039323-0308)"
246			workDir = r"C:\Documents and Settings\eborges\Desktop\Working"
247			#workDir = r"E:"
248			startDir = r"C:\Documents and Settings\eborges\Desktop\Working\STCL_EML_IMAGE_1101.zip"
249			Process(caseName, workDir, startDir)
250
251
252
253