Active_prgs/MCP/MCP_CopyUp_Request.py

"""


MCP_CopyUp_Request.py

Created by
Emanuel Borges  
08.14.2010

TODO:  Next make it gather the tar size and update your sheet on what's been uploaded. Also make it
automatically make the destination point based on the case you select and the date.

NOTE:  What if the ftp connection times out while it's being compressed...?

"""

#import win32api
import os, tarfile, time, ConcordanceHelperTools, NinoGenTools, shutil,zipfile,MCP_Lib
from directorySize2 import pretty_filesize2


def ReportAndStore(caseName, totalSize, reviewPlatform):
    reportSize = pretty_filesize2(totalSize)
    epoch = str(int(time.time()))
    #casesDir = r"C:\Documents and Settings\eborges\My Documents\My Dropbox\Documents\Cases"
    casesList,casesDir = MCP_Lib.GetCaseList()
    if casesDir:
        casePath = os.path.join(casesDir,caseName)
        outputFile = open(os.path.join(casePath,"UploadReport.txt"),'a')
        
        outputFile.write(epoch + " | " + reviewPlatform + " | " + time.strftime('%Y%m%d') + " | " +reportSize + "\n")
        outputFile.close()
    print 'MCP: Connecting to the Access Database...'
    clientMatter = caseName.split("_(")[1]
    clientMatter = clientMatter[:-1]
    clientMatter = clientMatter.replace('-','.')
    #accessDB = MCP_Lib.AccessDBConnection(r"W:\Manny\testing\TPM_CaseTracking.mdb")
    accessDB = MCP_Lib.AccessDBConnection(r"\\chiads01\app\DS_CaseTrack\TPM_CaseTracking.mdb")
    accessDB.UpdateCaseUpload(clientMatter,reviewPlatform,{epoch:(time.strftime('%m/%d/%Y'),reportSize)})
    accessDB.CloseAccessConnection()
    print 'MCP: Access database updated.'

def CompressSelected(pathToCompress, workDir, fileName):
    ## check available space. Acutally this could take too long to calculate the source media.
    ## warn them to check the available size, for now.
    #availSize = win32api.GetDiskFreeSpaceEx(rootPath)
    ## create the tar
    if os.path.splitext(fileName)[1].upper() in [".ZIP", ".RAR",".TAR",".7Z"]:
        print "MCP: It's already compressed. Skipping step."
        tFileName = pathToCompress
    else:
        #print "this is second workdir %s"% workDir
        tFileName = os.path.join(workDir , fileName) +".tar"
        tFile = tarfile.open(tFileName,'w')
        tFile.add(pathToCompress,recursive = True)
        tFile.close()
    return tFileName
    
def CopyUpRemote(compressedFile,newLocation,expeObj):
    newTargetPath = os.path.split(newLocation)[0]
    pathExists = expeObj.TestPath(newTargetPath)
    tempPath = newTargetPath
    pathRemainingList = []
    while pathExists == False:
        tempPath, tempPart = os.path.split(tempPath)
        pathRemainingList.append(tempPart)
        pathExists = expeObj.TestPath(tempPath)
    if pathRemainingList:
        pathRemainingList.reverse()
        for pathPiece in pathRemainingList:
            tempPath = os.path.join(tempPath,pathPiece)
            expeObj.CreateNewPath(tempPath)
        
        
    drive,null = os.path.splitdrive(compressedFile)
    if drive.upper() == "C:":
        print "MCP: moving up %s"%compressedFile
        print "MCP: to %s"% newLocation
        errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
        if errCode == True:
            os.remove(compressedFile)
    else:
        print "MCP: copying up %s"%compressedFile
        print "MCP: to %s"% newLocation
        errCode = expeObj.CopyToLN(compressedFile,newTargetPath)

def CopyUpMWESFTP(compressedFile,newLocation,mweSftpObj):
    newTargetPath = os.path.split(newLocation)[0]
    pathExists = mweSftpObj.TestPath(newTargetPath)
    if pathExists:
        pass
    else:
        mweSftpObj.CreateNewPath(newTargetPath)
        
    drive,null = os.path.splitdrive(compressedFile)
    if drive.upper() == "C:":
        print "MCP: moving up %s"%compressedFile
        print "MCP: to %s"% newLocation
        mweSftpObj.CopyToDis(compressedFile,newTargetPath)
        os.remove(compressedFile)
    else:
        print "MCP: copying up %s"%compressedFile
        print "MCP: to %s"% newLocation
        mweSftpObj.CopyToDis(compressedFile,newTargetPath)

def CopyUp(compressedFile, newLocation):
    ##  WARNING! needs to be changes so it increments a dir. What if this dir already exists.
    if os.path.exists(os.path.split(newLocation)[0]):
        pass
    else:
        os.makedirs(os.path.split(newLocation)[0])
    
    ## if the pre archive is on C, move it. Else just copy it.ie a dvd with a zip on it.
    drive,null = os.path.splitdrive(compressedFile)
    if drive.upper() == "C:":
        print "MCP: copying up %s"%compressedFile
        print "MCP: to %s"% newLocation
        os.rename(compressedFile, newLocation)
    else:
        shutil.copy2(compressedFile, newLocation)
        print "MCP: moving up %s"%compressedFile
        print "MCP: to %s"% newLocation
    

def Process(caseName, workDir, startDir, platform = 'Concordance DIS'):
    ##This should be external too.
    availableShare = "Dis29"
    expeObj = None
    mweSftpObj = None
    connectionTest = os.path.isdir(os.path.join(r'\\lisdisdss01',availableShare))
    if platform == 'Concordance DIS':
        mweSftpObj = MCP_Lib.MweFtpConnection()
    else:
        expeObj = MCP_Lib.ExpeDatConnection(platform)
    if connectionTest:
        totalSize = NinoGenTools.Counter()
        networkDir = GetCopyToLocation(caseName, availableShare,expeObj,platform)
        if os.path.isfile(startDir):
            startDir,indFile = os.path.split(startDir)
            compressList = [indFile]
        else:
            compressList = os.listdir(startDir)
        for item in compressList:
            if item.upper() == 'SYSTEM VOLUME INFORMATION':
                ## If it's the system folder skip it.
                ##  Consider changing this to a try or to a list of stuff to skip.
                pass
            else:
                pathToCompress = os.path.join(startDir,item)
                fileName = item
                ## TODO: check to see if the file already exists. If so, skip that request and give them a dialog.
                print "MCP: Compressing %s"% item
                compressedFile = CompressSelected(pathToCompress, workDir, fileName)
                #print compressedFile
                print "MCP: Done!"
                print "MCP: Gathering size..."
                if os.path.splitext(fileName)[1].upper() in [".ZIP"]:
                    print "MCP: It's a ZIP, so gathering uncompressed size..."
                    try:
                        tempZipSize = 0
                        tempZip = zipfile.ZipFile(compressedFile,'r')
                        for zipInfo in tempZip.infolist():
                            tempZipSize = tempZipSize + zipInfo.file_size
                        tempZip.close()
                        totalSize.inc(tempZipSize)
                        print "MCP: Uncompressed size gathered."
                    except:
                        print "MCP: Warning: Zip error. Defaulting to compressed size."
                        totalSize.inc(os.path.getsize(compressedFile)) 
                else:
                    totalSize.inc(os.path.getsize(compressedFile))
                    print "MCP: Size gathered."
                print "MCP: Copying up %s"% item
                #print "comressedFile is %s"% compressedFile
                ##  Copy or move it up.
                #CopyUp(compressedFile, os.path.join(networkDir+"\\"+time.strftime('%Y%m%d'), os.path.split(compressedFile)[1]))
                if platform == 'Concordance DIS':
                    ## SFTP coping is now only way.
                    CopyUpMWESFTP(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),mweSftpObj)
                    
                    ## Wan copying totally disabled.
                    ## CopyUp(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]))
                else:
                    CopyUpRemote(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),expeObj)
                print "MCP: Done!"
        print "MCP: Saving sizes..."
        ReportAndStore(caseName, totalSize.count, platform)
        print "MCP: Saving complete."
        print "MCP: All Items processed and copied!"
    else:
        print "MCP: ERROR!!! You are not connected to the DIS.\nPlease connect to the DIS and try your request again. \n\n\tEnd Of Line.\n"
    if mweSftpObj:
        mweSftpObj.CloseConnection()
    raw_input("\n\nMCP: Press Enter key to exit.\n\n\tEnd Of Line.\n")

def GetCopyToLocation(caseName, availableShare,expeObj,platform = 'Concordance DIS'):
    clientMatter = caseName.split("_(")[1]
    clientMatter = clientMatter[:-1]
    if platform == 'Concordance DIS':
        parsedPath = ConcordanceHelperTools.ParseClientMatter(clientMatter)
        parsedClientMatter = os.path.split(parsedPath)[1]
        copyToLocation = os.path.join(r"\\lisdisdss01\%s\C_D"%availableShare,parsedClientMatter)
        
        copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
        
        ## Check to see if this path already exists.  If it does, keep incrementing it until it dosent.
        dirIncrement = NinoGenTools.Counter(1)
        if os.path.exists(copyToLocation):
            copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
        while os.path.exists(copyToLocation):
            copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
            
        ## Now that it's SFTP only, remove the part before c_d
        copyToLocation = copyToLocation.split("\\\\lisdisdss01\\%s\\"%availableShare)[1]
    else:
        parsedPath = ConcordanceHelperTools.ParseClientMatterRemote(clientMatter)
        copyToLocation = parsedPath
        
        copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
        
        ## Check to see if this path already exists.  If it does, keep incrementing it until it dosent.
        dirIncrement = NinoGenTools.Counter(1)
        if expeObj.TestPath(copyToLocation) == True:
            copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
        while expeObj.TestPath(copyToLocation) == True:
            copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
            dirIncrement.inc()
            
    return copyToLocation

if __name__ == '__main__':
    caseName = "DaVita-Dallas_(039323-0308)"
    workDir = r"C:\Documents and Settings\eborges\Desktop\Working"
    #workDir = r"E:"
    startDir = r"C:\Documents and Settings\eborges\Desktop\Working\STCL_EML_IMAGE_1101.zip"
    Process(caseName, workDir, startDir)
    
    
Revision:	8
Committed:	Sat May 5 04:21:19 2012 UTC (13 years, 10 months ago) by ninoborges
Content type:	text/x-python
File size:	10974 byte(s)
Log Message:	Initial Import
#	User	Rev	Content
1	ninoborges	8	"""
2
3
4			MCP_CopyUp_Request.py
5
6			Created by
7			Emanuel Borges
8			08.14.2010
9
10			TODO: Next make it gather the tar size and update your sheet on what's been uploaded. Also make it
11			automatically make the destination point based on the case you select and the date.
12
13			NOTE: What if the ftp connection times out while it's being compressed...?
14
15			"""
16
17			#import win32api
18			import os, tarfile, time, ConcordanceHelperTools, NinoGenTools, shutil,zipfile,MCP_Lib
19			from directorySize2 import pretty_filesize2
20
21
22			def ReportAndStore(caseName, totalSize, reviewPlatform):
23			reportSize = pretty_filesize2(totalSize)
24			epoch = str(int(time.time()))
25			#casesDir = r"C:\Documents and Settings\eborges\My Documents\My Dropbox\Documents\Cases"
26			casesList,casesDir = MCP_Lib.GetCaseList()
27			if casesDir:
28			casePath = os.path.join(casesDir,caseName)
29			outputFile = open(os.path.join(casePath,"UploadReport.txt"),'a')
30
31			outputFile.write(epoch + " \| " + reviewPlatform + " \| " + time.strftime('%Y%m%d') + " \| " +reportSize + "\n")
32			outputFile.close()
33			print 'MCP: Connecting to the Access Database...'
34			clientMatter = caseName.split("_(")[1]
35			clientMatter = clientMatter[:-1]
36			clientMatter = clientMatter.replace('-','.')
37			#accessDB = MCP_Lib.AccessDBConnection(r"W:\Manny\testing\TPM_CaseTracking.mdb")
38			accessDB = MCP_Lib.AccessDBConnection(r"\\chiads01\app\DS_CaseTrack\TPM_CaseTracking.mdb")
39			accessDB.UpdateCaseUpload(clientMatter,reviewPlatform,{epoch:(time.strftime('%m/%d/%Y'),reportSize)})
40			accessDB.CloseAccessConnection()
41			print 'MCP: Access database updated.'
42
43			def CompressSelected(pathToCompress, workDir, fileName):
44			## check available space. Acutally this could take too long to calculate the source media.
45			## warn them to check the available size, for now.
46			#availSize = win32api.GetDiskFreeSpaceEx(rootPath)
47			## create the tar
48			if os.path.splitext(fileName)[1].upper() in [".ZIP", ".RAR",".TAR",".7Z"]:
49			print "MCP: It's already compressed. Skipping step."
50			tFileName = pathToCompress
51			else:
52			#print "this is second workdir %s"% workDir
53			tFileName = os.path.join(workDir , fileName) +".tar"
54			tFile = tarfile.open(tFileName,'w')
55			tFile.add(pathToCompress,recursive = True)
56			tFile.close()
57			return tFileName
58
59			def CopyUpRemote(compressedFile,newLocation,expeObj):
60			newTargetPath = os.path.split(newLocation)[0]
61			pathExists = expeObj.TestPath(newTargetPath)
62			tempPath = newTargetPath
63			pathRemainingList = []
64			while pathExists == False:
65			tempPath, tempPart = os.path.split(tempPath)
66			pathRemainingList.append(tempPart)
67			pathExists = expeObj.TestPath(tempPath)
68			if pathRemainingList:
69			pathRemainingList.reverse()
70			for pathPiece in pathRemainingList:
71			tempPath = os.path.join(tempPath,pathPiece)
72			expeObj.CreateNewPath(tempPath)
73
74
75			drive,null = os.path.splitdrive(compressedFile)
76			if drive.upper() == "C:":
77			print "MCP: moving up %s"%compressedFile
78			print "MCP: to %s"% newLocation
79			errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
80			if errCode == True:
81			os.remove(compressedFile)
82			else:
83			print "MCP: copying up %s"%compressedFile
84			print "MCP: to %s"% newLocation
85			errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
86
87			def CopyUpMWESFTP(compressedFile,newLocation,mweSftpObj):
88			newTargetPath = os.path.split(newLocation)[0]
89			pathExists = mweSftpObj.TestPath(newTargetPath)
90			if pathExists:
91			pass
92			else:
93			mweSftpObj.CreateNewPath(newTargetPath)
94
95			drive,null = os.path.splitdrive(compressedFile)
96			if drive.upper() == "C:":
97			print "MCP: moving up %s"%compressedFile
98			print "MCP: to %s"% newLocation
99			mweSftpObj.CopyToDis(compressedFile,newTargetPath)
100			os.remove(compressedFile)
101			else:
102			print "MCP: copying up %s"%compressedFile
103			print "MCP: to %s"% newLocation
104			mweSftpObj.CopyToDis(compressedFile,newTargetPath)
105
106			def CopyUp(compressedFile, newLocation):
107			## WARNING! needs to be changes so it increments a dir. What if this dir already exists.
108			if os.path.exists(os.path.split(newLocation)[0]):
109			pass
110			else:
111			os.makedirs(os.path.split(newLocation)[0])
112
113			## if the pre archive is on C, move it. Else just copy it.ie a dvd with a zip on it.
114			drive,null = os.path.splitdrive(compressedFile)
115			if drive.upper() == "C:":
116			print "MCP: copying up %s"%compressedFile
117			print "MCP: to %s"% newLocation
118			os.rename(compressedFile, newLocation)
119			else:
120			shutil.copy2(compressedFile, newLocation)
121			print "MCP: moving up %s"%compressedFile
122			print "MCP: to %s"% newLocation
123
124
125			def Process(caseName, workDir, startDir, platform = 'Concordance DIS'):
126			##This should be external too.
127			availableShare = "Dis29"
128			expeObj = None
129			mweSftpObj = None
130			connectionTest = os.path.isdir(os.path.join(r'\\lisdisdss01',availableShare))
131			if platform == 'Concordance DIS':
132			mweSftpObj = MCP_Lib.MweFtpConnection()
133			else:
134			expeObj = MCP_Lib.ExpeDatConnection(platform)
135			if connectionTest:
136			totalSize = NinoGenTools.Counter()
137			networkDir = GetCopyToLocation(caseName, availableShare,expeObj,platform)
138			if os.path.isfile(startDir):
139			startDir,indFile = os.path.split(startDir)
140			compressList = [indFile]
141			else:
142			compressList = os.listdir(startDir)
143			for item in compressList:
144			if item.upper() == 'SYSTEM VOLUME INFORMATION':
145			## If it's the system folder skip it.
146			## Consider changing this to a try or to a list of stuff to skip.
147			pass
148			else:
149			pathToCompress = os.path.join(startDir,item)
150			fileName = item
151			## TODO: check to see if the file already exists. If so, skip that request and give them a dialog.
152			print "MCP: Compressing %s"% item
153			compressedFile = CompressSelected(pathToCompress, workDir, fileName)
154			#print compressedFile
155			print "MCP: Done!"
156			print "MCP: Gathering size..."
157			if os.path.splitext(fileName)[1].upper() in [".ZIP"]:
158			print "MCP: It's a ZIP, so gathering uncompressed size..."
159			try:
160			tempZipSize = 0
161			tempZip = zipfile.ZipFile(compressedFile,'r')
162			for zipInfo in tempZip.infolist():
163			tempZipSize = tempZipSize + zipInfo.file_size
164			tempZip.close()
165			totalSize.inc(tempZipSize)
166			print "MCP: Uncompressed size gathered."
167			except:
168			print "MCP: Warning: Zip error. Defaulting to compressed size."
169			totalSize.inc(os.path.getsize(compressedFile))
170			else:
171			totalSize.inc(os.path.getsize(compressedFile))
172			print "MCP: Size gathered."
173			print "MCP: Copying up %s"% item
174			#print "comressedFile is %s"% compressedFile
175			## Copy or move it up.
176			#CopyUp(compressedFile, os.path.join(networkDir+"\\"+time.strftime('%Y%m%d'), os.path.split(compressedFile)[1]))
177			if platform == 'Concordance DIS':
178			## SFTP coping is now only way.
179			CopyUpMWESFTP(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),mweSftpObj)
180
181			## Wan copying totally disabled.
182			## CopyUp(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]))
183			else:
184			CopyUpRemote(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),expeObj)
185			print "MCP: Done!"
186			print "MCP: Saving sizes..."
187			ReportAndStore(caseName, totalSize.count, platform)
188			print "MCP: Saving complete."
189			print "MCP: All Items processed and copied!"
190			else:
191			print "MCP: ERROR!!! You are not connected to the DIS.\nPlease connect to the DIS and try your request again. \n\n\tEnd Of Line.\n"
192			if mweSftpObj:
193			mweSftpObj.CloseConnection()
194			raw_input("\n\nMCP: Press Enter key to exit.\n\n\tEnd Of Line.\n")
195
196			def GetCopyToLocation(caseName, availableShare,expeObj,platform = 'Concordance DIS'):
197			clientMatter = caseName.split("_(")[1]
198			clientMatter = clientMatter[:-1]
199			if platform == 'Concordance DIS':
200			parsedPath = ConcordanceHelperTools.ParseClientMatter(clientMatter)
201			parsedClientMatter = os.path.split(parsedPath)[1]
202			copyToLocation = os.path.join(r"\\lisdisdss01\%s\C_D"%availableShare,parsedClientMatter)
203
204			copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
205
206			## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
207			dirIncrement = NinoGenTools.Counter(1)
208			if os.path.exists(copyToLocation):
209			copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
210			dirIncrement.inc()
211			while os.path.exists(copyToLocation):
212			copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
213			dirIncrement.inc()
214
215			## Now that it's SFTP only, remove the part before c_d
216			copyToLocation = copyToLocation.split("\\\\lisdisdss01\\%s\\"%availableShare)[1]
217			else:
218			parsedPath = ConcordanceHelperTools.ParseClientMatterRemote(clientMatter)
219			copyToLocation = parsedPath
220
221			copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
222
223			## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
224			dirIncrement = NinoGenTools.Counter(1)
225			if expeObj.TestPath(copyToLocation) == True:
226			copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
227			dirIncrement.inc()
228			while expeObj.TestPath(copyToLocation) == True:
229			copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
230			dirIncrement.inc()
231
232			return copyToLocation
233
234			if __name__ == '__main__':
235			caseName = "DaVita-Dallas_(039323-0308)"
236			workDir = r"C:\Documents and Settings\eborges\Desktop\Working"
237			#workDir = r"E:"
238			startDir = r"C:\Documents and Settings\eborges\Desktop\Working\STCL_EML_IMAGE_1101.zip"
239			Process(caseName, workDir, startDir)
240
241
242
243