| 1 |
"""
|
| 2 |
|
| 3 |
|
| 4 |
MCP_CopyUp_Request.py
|
| 5 |
|
| 6 |
Created by
|
| 7 |
Emanuel Borges
|
| 8 |
08.14.2010
|
| 9 |
|
| 10 |
TODO: Next make it gather the tar size and update your sheet on what's been uploaded. Also make it
|
| 11 |
automatically make the destination point based on the case you select and the date.
|
| 12 |
|
| 13 |
NOTE: What if the ftp connection times out while it's being compressed...?
|
| 14 |
|
| 15 |
"""
|
| 16 |
|
| 17 |
#import win32api
|
| 18 |
import os, tarfile, time, ConcordanceHelperTools, NinoGenTools, shutil,zipfile,MCP_Lib
|
| 19 |
from directorySize2 import pretty_filesize2
|
| 20 |
|
| 21 |
|
| 22 |
def RegisterUpload(workDir,fullUploadPath,caseName,expeObj):
|
| 23 |
"""This will register the upload in a tracking file, so we can make sure copyups are being removed. IDS only."""
|
| 24 |
## Test to make sure this knows that it's the archive that needs to be removed and not real file.
|
| 25 |
uploadRegFileName = epoch = str(int(time.time()))
|
| 26 |
uploadRegFile = open(os.path.join(workDir,uploadRegFileName+".txt"),'w')
|
| 27 |
uploadRegFile.write(caseName + "|" + fullUploadPath + "\n")
|
| 28 |
uploadRegFile.close()
|
| 29 |
|
| 30 |
uploadRegFilePath = os.path.join(workDir,uploadRegFileName+".txt")
|
| 31 |
print uploadRegFilePath
|
| 32 |
newTargetPath = "Uploads/Upload_Tracking"
|
| 33 |
errCode = expeObj.CopyToLN(uploadRegFilePath,newTargetPath)
|
| 34 |
print errCode
|
| 35 |
if errCode == True:
|
| 36 |
os.remove(uploadRegFilePath)
|
| 37 |
|
| 38 |
|
| 39 |
def ReportAndStore(caseName, totalSize, reviewPlatform):
|
| 40 |
reportSize = pretty_filesize2(totalSize)
|
| 41 |
epoch = str(int(time.time()))
|
| 42 |
#casesDir = r"C:\Documents and Settings\eborges\My Documents\My Dropbox\Documents\Cases"
|
| 43 |
casesList,myActiveCases,officeCases, allCases, casesDir,casePathStyleMatrix = MCP_Lib.GetCaseList()
|
| 44 |
if caseName in casesList:
|
| 45 |
if casesDir:
|
| 46 |
casePath = os.path.join(casesDir,caseName)
|
| 47 |
outputFile = open(os.path.join(casePath,"UploadReport.txt"),'a')
|
| 48 |
|
| 49 |
outputFile.write(epoch + " | " + reviewPlatform + " | " + time.strftime('%Y%m%d') + " | " +reportSize + "\n")
|
| 50 |
outputFile.close()
|
| 51 |
print 'MCP: Connecting to the Access Database...'
|
| 52 |
clientMatter = caseName.split("_(")[1]
|
| 53 |
clientMatter = clientMatter[:-1]
|
| 54 |
clientMatter = clientMatter.replace('-','.')
|
| 55 |
#accessDB = MCP_Lib.AccessDBConnection(r"W:\Manny\testing\TPM_CaseTracking.mdb")
|
| 56 |
accessDB = MCP_Lib.AccessDBConnection(r"\\chiads01\app\DS_CaseTrack\TPM_CaseTracking.mdb")
|
| 57 |
accessDB.UpdateCaseUpload(clientMatter,reviewPlatform,{epoch:(time.strftime('%m/%d/%Y'),reportSize)})
|
| 58 |
accessDB.CloseAccessConnection()
|
| 59 |
print 'MCP: Access database updated.'
|
| 60 |
|
| 61 |
def CompressSelected(pathToCompress, workDir, fileName):
|
| 62 |
## check available space. Acutally this could take too long to calculate the source media.
|
| 63 |
## warn them to check the available size, for now.
|
| 64 |
#availSize = win32api.GetDiskFreeSpaceEx(rootPath)
|
| 65 |
## create the tar
|
| 66 |
if os.path.splitext(fileName)[1].upper() in MCP_Lib.GetArchiveFileTypeList():#[".ZIP", ".RAR",".TAR",".7Z"]:
|
| 67 |
print "MCP: It's already compressed. Skipping step."
|
| 68 |
tFileName = pathToCompress
|
| 69 |
else:
|
| 70 |
#print "this is second workdir %s"% workDir
|
| 71 |
tFileName = os.path.join(workDir , fileName) +".tar"
|
| 72 |
tFile = tarfile.open(tFileName,'w')
|
| 73 |
tFile.add(pathToCompress,arcname = fileName,recursive = True)
|
| 74 |
tFile.close()
|
| 75 |
return tFileName
|
| 76 |
|
| 77 |
def CopyLNToLN(sourceLocationAndFile,newLocation,expeObj):
|
| 78 |
## Here new location does not have a file ending.
|
| 79 |
pathExists = expeObj.TestPath(newLocation)
|
| 80 |
tempPath = newLocation
|
| 81 |
pathRemainingList = []
|
| 82 |
while pathExists == False:
|
| 83 |
tempPath, tempPart = os.path.split(tempPath)
|
| 84 |
pathRemainingList.append(tempPart)
|
| 85 |
pathExists = expeObj.TestPath(tempPath)
|
| 86 |
if pathRemainingList:
|
| 87 |
pathRemainingList.reverse()
|
| 88 |
#print pathRemainingList
|
| 89 |
for pathPiece in pathRemainingList:
|
| 90 |
tempPath = os.path.join(tempPath,pathPiece)
|
| 91 |
expeObj.CreateNewPath(tempPath)
|
| 92 |
|
| 93 |
expeObj.MoveOnLN(sourceLocationAndFile,os.path.join(newLocation,os.path.split(sourceLocationAndFile)[1]))
|
| 94 |
print "the file would have been moved to\n%s"%os.path.join(newLocation,os.path.split(sourceLocationAndFile)[1])
|
| 95 |
|
| 96 |
|
| 97 |
def CopyUpRemote(compressedFile,newLocation,expeObj):
|
| 98 |
newTargetPath = os.path.split(newLocation)[0]
|
| 99 |
pathExists = expeObj.TestPath(newTargetPath)
|
| 100 |
tempPath = newTargetPath
|
| 101 |
pathRemainingList = []
|
| 102 |
while pathExists == False:
|
| 103 |
tempPath, tempPart = os.path.split(tempPath)
|
| 104 |
pathRemainingList.append(tempPart)
|
| 105 |
pathExists = expeObj.TestPath(tempPath)
|
| 106 |
if pathRemainingList:
|
| 107 |
pathRemainingList.reverse()
|
| 108 |
for pathPiece in pathRemainingList:
|
| 109 |
tempPath = os.path.join(tempPath,pathPiece)
|
| 110 |
expeObj.CreateNewPath(tempPath)
|
| 111 |
|
| 112 |
|
| 113 |
drive,null = os.path.splitdrive(compressedFile)
|
| 114 |
if drive.upper() == "C:":
|
| 115 |
print "MCP: moving up %s"%compressedFile
|
| 116 |
print "MCP: to %s"% newLocation
|
| 117 |
errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
|
| 118 |
if errCode == True:
|
| 119 |
os.remove(compressedFile)
|
| 120 |
else:
|
| 121 |
print "MCP: copying up %s"%compressedFile
|
| 122 |
print "MCP: to %s"% newLocation
|
| 123 |
errCode = expeObj.CopyToLN(compressedFile,newTargetPath)
|
| 124 |
|
| 125 |
def CopyUpMWESFTP(compressedFile,newLocation,mweSftpObj):
|
| 126 |
newTargetPath = os.path.split(newLocation)[0]
|
| 127 |
pathExists = mweSftpObj.TestPath(newTargetPath)
|
| 128 |
if pathExists:
|
| 129 |
pass
|
| 130 |
else:
|
| 131 |
mweSftpObj.CreateNewPath(newTargetPath)
|
| 132 |
|
| 133 |
drive,null = os.path.splitdrive(compressedFile)
|
| 134 |
if drive.upper() == "C:":
|
| 135 |
print "MCP: moving up %s"%compressedFile
|
| 136 |
print "MCP: to %s"% newLocation
|
| 137 |
mweSftpObj.CopyToDis(compressedFile,newTargetPath)
|
| 138 |
os.remove(compressedFile)
|
| 139 |
else:
|
| 140 |
print "MCP: copying up %s"%compressedFile
|
| 141 |
print "MCP: to %s"% newLocation
|
| 142 |
mweSftpObj.CopyToDis(compressedFile,newTargetPath)
|
| 143 |
|
| 144 |
def CopyUp(compressedFile, newLocation):
|
| 145 |
## WARNING! needs to be changes so it increments a dir. What if this dir already exists.
|
| 146 |
if os.path.exists(os.path.split(newLocation)[0]):
|
| 147 |
pass
|
| 148 |
else:
|
| 149 |
os.makedirs(os.path.split(newLocation)[0])
|
| 150 |
|
| 151 |
## if the pre archive is on C, move it. Else just copy it.ie a dvd with a zip on it.
|
| 152 |
drive,null = os.path.splitdrive(compressedFile)
|
| 153 |
if drive.upper() == "C:":
|
| 154 |
print "MCP: copying up %s"%compressedFile
|
| 155 |
print "MCP: to %s"% newLocation
|
| 156 |
os.rename(compressedFile, newLocation)
|
| 157 |
else:
|
| 158 |
shutil.copy2(compressedFile, newLocation)
|
| 159 |
print "MCP: moving up %s"%compressedFile
|
| 160 |
print "MCP: to %s"% newLocation
|
| 161 |
|
| 162 |
|
| 163 |
def Process(caseName, workDir, startDir, platform = 'Concordance DIS', sourceMediaLocation = 'local', uploadType = 'Generic (Old Cases Only)'):
|
| 164 |
##This should be external too.
|
| 165 |
availableShare = "Dis29"
|
| 166 |
expeObj = None
|
| 167 |
mweSftpObj = None
|
| 168 |
if sourceMediaLocation == 'local':
|
| 169 |
if platform == 'Concordance DIS':
|
| 170 |
mweSftpObj = MCP_Lib.MweFtpConnection()
|
| 171 |
connectionTest = os.path.isdir(os.path.join(r'\\lisdisdss01',availableShare))
|
| 172 |
else:
|
| 173 |
expeObj = MCP_Lib.ExpeDatConnection(platform)
|
| 174 |
connectionTest = True
|
| 175 |
if connectionTest:
|
| 176 |
totalSize = NinoGenTools.Counter()
|
| 177 |
networkDir = GetCopyToLocation(caseName, availableShare,expeObj,platform,uploadType)
|
| 178 |
if os.path.isfile(startDir):
|
| 179 |
startDir,indFile = os.path.split(startDir)
|
| 180 |
compressList = [indFile]
|
| 181 |
else:
|
| 182 |
compressList = os.listdir(startDir)
|
| 183 |
for item in compressList:
|
| 184 |
if item.upper() == 'SYSTEM VOLUME INFORMATION':
|
| 185 |
## If it's the system folder skip it.
|
| 186 |
## Consider changing this to a try or to a list of stuff to skip.
|
| 187 |
pass
|
| 188 |
else:
|
| 189 |
pathToCompress = os.path.join(startDir,item)
|
| 190 |
fileName = item
|
| 191 |
## TODO: check to see if the file already exists. If so, skip that request and give them a dialog.
|
| 192 |
print "MCP: Compressing %s"% item
|
| 193 |
compressedFile = CompressSelected(pathToCompress, workDir, fileName)
|
| 194 |
#print compressedFile
|
| 195 |
print "MCP: Done!"
|
| 196 |
print "MCP: Gathering size..."
|
| 197 |
if os.path.splitext(fileName)[1].upper() in [".ZIP"]:
|
| 198 |
print "MCP: It's a ZIP, so gathering uncompressed size..."
|
| 199 |
try:
|
| 200 |
tempZipSize = 0
|
| 201 |
tempZip = zipfile.ZipFile(compressedFile,'r')
|
| 202 |
for zipInfo in tempZip.infolist():
|
| 203 |
tempZipSize = tempZipSize + zipInfo.file_size
|
| 204 |
tempZip.close()
|
| 205 |
totalSize.inc(tempZipSize)
|
| 206 |
print "MCP: Uncompressed size gathered."
|
| 207 |
except:
|
| 208 |
print "MCP: Warning: Zip error. Defaulting to compressed size."
|
| 209 |
totalSize.inc(os.path.getsize(compressedFile))
|
| 210 |
else:
|
| 211 |
totalSize.inc(os.path.getsize(compressedFile))
|
| 212 |
print "MCP: Size gathered."
|
| 213 |
print "MCP: Copying up %s"% item
|
| 214 |
#print "comressedFile is %s"% compressedFile
|
| 215 |
## Copy or move it up.
|
| 216 |
#CopyUp(compressedFile, os.path.join(networkDir+"\\"+time.strftime('%Y%m%d'), os.path.split(compressedFile)[1]))
|
| 217 |
if platform == 'Concordance DIS':
|
| 218 |
## SFTP coping is now only way.
|
| 219 |
CopyUpMWESFTP(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),mweSftpObj)
|
| 220 |
|
| 221 |
## Wan copying totally disabled.
|
| 222 |
## CopyUp(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]))
|
| 223 |
else:
|
| 224 |
CopyUpRemote(compressedFile, os.path.join(networkDir, os.path.split(compressedFile)[1]),expeObj)
|
| 225 |
RegisterUpload(workDir,os.path.join(networkDir, os.path.split(compressedFile)[1]),caseName,expeObj)
|
| 226 |
print "MCP: Done!"
|
| 227 |
print "MCP: Saving sizes..."
|
| 228 |
ReportAndStore(caseName, totalSize.count, platform)
|
| 229 |
print "MCP: Saving complete."
|
| 230 |
print "MCP: All Items processed and copied!"
|
| 231 |
else:
|
| 232 |
print "MCP: ERROR!!! You are not connected to the DIS.\nPlease connect to the DIS and try your request again. \n\n\tEnd Of Line.\n"
|
| 233 |
else:
|
| 234 |
## User is copying from LN to LN
|
| 235 |
expeObj = MCP_Lib.ExpeDatConnection(platform)
|
| 236 |
networkDir = GetCopyToLocation(caseName, availableShare,expeObj,platform,uploadType)
|
| 237 |
print "MCP: Gathering size..."
|
| 238 |
totalSize = expeObj.GatherSize(os.path.join('Uploads',startDir))
|
| 239 |
print "MCP: Size gathered."
|
| 240 |
print "MCP: Moving up %s"% os.path.join('Uploads',startDir)
|
| 241 |
## need to change this to put the file name or folder after networkDir
|
| 242 |
CopyLNToLN(os.path.join('Uploads',startDir),networkDir,expeObj)
|
| 243 |
RegisterUpload(workDir,networkDir,caseName,expeObj)
|
| 244 |
#expeObj.MoveOnLN(os.path.join('Uploads',startDir),os.path.join(networkDir,os.path.split(startDir)[1]))
|
| 245 |
#print "the file would have been moved to\n%s"%os.path.join(networkDir,os.path.split(startDir)[1])
|
| 246 |
print "MCP: Done!"
|
| 247 |
#print totalSize
|
| 248 |
print "MCP: Saving sizes..."
|
| 249 |
ReportAndStore(caseName, int(totalSize), platform)
|
| 250 |
print "MCP: Saving complete."
|
| 251 |
## Verify that it's there
|
| 252 |
print "MCP: All Items processed and copied!"
|
| 253 |
|
| 254 |
if mweSftpObj:
|
| 255 |
mweSftpObj.CloseConnection()
|
| 256 |
raw_input("\n\nMCP: Press Enter key to exit.\n\n\tEnd Of Line.\n")
|
| 257 |
|
| 258 |
def GetCopyToLocation(caseName, availableShare,expeObj,platform = 'Concordance DIS', uploadType = 'Generic (Old Cases Only)'):
|
| 259 |
clientMatter = caseName.split("_(")[1]
|
| 260 |
clientMatter = clientMatter[:-1]
|
| 261 |
if platform == 'Concordance DIS':
|
| 262 |
parsedPath = ConcordanceHelperTools.ParseClientMatter(clientMatter)
|
| 263 |
parsedClientMatter = os.path.split(parsedPath)[1]
|
| 264 |
copyToLocation = os.path.join(r"\\lisdisdss01\%s\C_D"%availableShare,parsedClientMatter)
|
| 265 |
|
| 266 |
copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
|
| 267 |
|
| 268 |
## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
|
| 269 |
dirIncrement = NinoGenTools.Counter(1)
|
| 270 |
if os.path.exists(copyToLocation):
|
| 271 |
copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
|
| 272 |
dirIncrement.inc()
|
| 273 |
while os.path.exists(copyToLocation):
|
| 274 |
copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
|
| 275 |
dirIncrement.inc()
|
| 276 |
|
| 277 |
## Now that it's SFTP only, remove the part before c_d
|
| 278 |
copyToLocation = copyToLocation.split("\\\\lisdisdss01\\%s\\"%availableShare)[1]
|
| 279 |
else:
|
| 280 |
uploadTypeMatrix = {'Processed Review Data':'Processed_Data\\rev','Hard Copy Review Data':'Hard_Copy\\HC','Loose Unprocessed Review Data':'Loose\\loose','Subset Tiffs':'TIFs\\subset','Other Party Production Data':'Productions\\Other_Party\\<Party_Name>','Generic (Old Cases Only)':'Working' }
|
| 281 |
parsedPath = ConcordanceHelperTools.ParseClientMatterRemote(clientMatter)
|
| 282 |
copyToLocation = parsedPath
|
| 283 |
## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
|
| 284 |
dirIncrement = NinoGenTools.Counter(1)
|
| 285 |
|
| 286 |
if uploadType == 'Generic (Old Cases Only)':
|
| 287 |
#copyToLocation = os.path.join(os.path.join(copyToLocation,uploadTypeMatrix[uploadType]),time.strftime('%Y%m%d'))
|
| 288 |
copyToLocation = os.path.join(copyToLocation,time.strftime('%Y%m%d'))
|
| 289 |
if expeObj.TestPath(copyToLocation) == True:
|
| 290 |
copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
|
| 291 |
dirIncrement.inc()
|
| 292 |
else:
|
| 293 |
copyToLocation = os.path.join(copyToLocation,uploadTypeMatrix[uploadType]+'_0001')
|
| 294 |
|
| 295 |
## Check to see if this path already exists. If it does, keep incrementing it until it dosent.
|
| 296 |
#dirIncrement = NinoGenTools.Counter(1)
|
| 297 |
## For new version of MCP, for it to 0001
|
| 298 |
#if expeObj.TestPath(copyToLocation) == True:
|
| 299 |
# copyToLocation = copyToLocation+ "_"+"%0*d"%(4,dirIncrement.count)
|
| 300 |
# dirIncrement.inc()
|
| 301 |
while expeObj.TestPath(copyToLocation) == True:
|
| 302 |
copyToLocation = copyToLocation[:-4] +"%0*d"%(4,dirIncrement.count)
|
| 303 |
dirIncrement.inc()
|
| 304 |
|
| 305 |
return copyToLocation
|
| 306 |
|
| 307 |
if __name__ == '__main__':
|
| 308 |
caseName = "DaVita-Dallas_(039323-0308)"
|
| 309 |
workDir = r"C:\Documents and Settings\eborges\Desktop\Working"
|
| 310 |
#workDir = r"E:"
|
| 311 |
startDir = r"C:\Documents and Settings\eborges\Desktop\Working\STCL_EML_IMAGE_1101.zip"
|
| 312 |
Process(caseName, workDir, startDir)
|
| 313 |
|
| 314 |
|
| 315 |
|
| 316 |
|
