| 1 |
nino.borges |
871 |
"""
|
| 2 |
|
|
|
| 3 |
|
|
Amazon-AnalyzeNamesDeepNormOutput
|
| 4 |
|
|
|
| 5 |
|
|
Created by:
|
| 6 |
|
|
Emanuel Borges
|
| 7 |
|
|
12.17.2024
|
| 8 |
|
|
|
| 9 |
|
|
This program will assist with analyzing the more complex 'deep' output logs from the Names Norm program. This will take in all of the vals output files and the priv log,
|
| 10 |
nino.borges |
917 |
and will compare the high confidence downgrades and upgrades to determine, if these were changed, if there are any surviving attorney values in the legalSource field.
|
| 11 |
nino.borges |
871 |
|
| 12 |
|
|
"""
|
| 13 |
|
|
|
| 14 |
|
|
|
| 15 |
|
|
import os
|
| 16 |
|
|
|
| 17 |
nino.borges |
917 |
version = '0.2.0'
|
| 18 |
|
|
|
| 19 |
nino.borges |
871 |
def FieldFullValueDedupe(valuesList):
|
| 20 |
|
|
"""Attempts to deduplicate a list of values from a specific field using the FULL VALUE. This was created because there appears to be duplicate values int he formatted fields.
|
| 21 |
|
|
returns a new set with just the cleaned and deduplicated values."""
|
| 22 |
|
|
## Going to do this the long way because of possible uppercase-lowercase issues. These should all be uppercase but there shouldnt have been dups either...
|
| 23 |
|
|
newSet = set()
|
| 24 |
|
|
for item in valuesList:
|
| 25 |
|
|
item = item.strip()
|
| 26 |
|
|
newSet.add(item.upper())
|
| 27 |
|
|
return newSet
|
| 28 |
|
|
|
| 29 |
|
|
|
| 30 |
|
|
if __name__ == '__main__':
|
| 31 |
|
|
## VEAS
|
| 32 |
|
|
valsFilesToIngestList = ['NameNormDeepOutputText(ToVals).txt','NameNormDeepOutputText(ccVals).txt', 'NameNormDeepOutputText(fromVals).txt',
|
| 33 |
|
|
'NameNormDeepOutputText(authorValue).txt']
|
| 34 |
|
|
pathToValsFiles = r"C:\Test_Dir\Amazon\VEAS-CAAG-20241204\Testing3"
|
| 35 |
|
|
privLogFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Testing_3\PrivLogExport_20241204_VEAS_Converted.txt"
|
| 36 |
|
|
legalSourcePosition = 40
|
| 37 |
|
|
|
| 38 |
|
|
|
| 39 |
|
|
## CAAG
|
| 40 |
|
|
## valsFilesToIngestList = ['NameNormDeepOutputText(bccVals).txt', 'NameNormDeepOutputText(ToVals).txt','NameNormDeepOutputText(ccVals).txt', 'NameNormDeepOutputText(fromVals).txt',
|
| 41 |
|
|
## 'NameNormDeepOutputText(docAuthor).txt']
|
| 42 |
|
|
## pathToValsFiles = r"C:\Test_Dir\Amazon\20241215\20241211_CAAG"
|
| 43 |
|
|
## privLogFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\20241215\PrivLogExports\PrivLogExport_20241211_CAAG_Converted.txt"
|
| 44 |
|
|
## legalSourcePosition = 45
|
| 45 |
|
|
|
| 46 |
|
|
outputFile = open(r"C:\Test_Dir\Amazon\deepAnalysisTesting.txt",'w')
|
| 47 |
|
|
|
| 48 |
|
|
highProfileMatrix = {}
|
| 49 |
|
|
|
| 50 |
|
|
|
| 51 |
|
|
## First ingest any high profile downgrades into the matrix.
|
| 52 |
|
|
for fName in valsFilesToIngestList:
|
| 53 |
|
|
contents = open(os.path.join(pathToValsFiles,fName)).readlines()
|
| 54 |
|
|
for line in contents:
|
| 55 |
|
|
line = line.replace("\n","")
|
| 56 |
|
|
docID, potIssues = line.split("|")
|
| 57 |
|
|
potIssues = potIssues.split(";")
|
| 58 |
|
|
for potIssue in potIssues:
|
| 59 |
|
|
if "High Confidence Potential downgrade" in potIssue:
|
| 60 |
|
|
attnyVal = potIssue.split("(")[0]
|
| 61 |
|
|
attnyVal = attnyVal.strip()
|
| 62 |
|
|
#attnyVal = attnyVal + ")"
|
| 63 |
|
|
try:
|
| 64 |
|
|
highProfileMatrix[docID].add(attnyVal.upper())
|
| 65 |
|
|
print(f"adding another {highProfileMatrix[docID]}, {docID}")
|
| 66 |
|
|
except KeyError:
|
| 67 |
|
|
highProfileMatrix[docID] = set()
|
| 68 |
|
|
highProfileMatrix[docID].add(attnyVal.upper())
|
| 69 |
|
|
|
| 70 |
|
|
## Second, for each line in the privLog, determine if removing the names from legalSource will result in no surviving attorney values for that doc.
|
| 71 |
|
|
contents = open(privLogFileName, encoding = 'UTF-8').readlines()
|
| 72 |
|
|
headerRow = contents[0]
|
| 73 |
|
|
headerRow = headerRow.split("|")
|
| 74 |
|
|
contents = contents[1:]
|
| 75 |
|
|
print(f"The field {headerRow[legalSourcePosition]} will be used for the legal source field.")
|
| 76 |
|
|
print(f"There are {len(contents)} to test.")
|
| 77 |
|
|
|
| 78 |
|
|
for line in contents:
|
| 79 |
|
|
line = line.replace("\n","")
|
| 80 |
|
|
line = line.split("|")
|
| 81 |
|
|
docID = line[0]
|
| 82 |
|
|
if docID in list(highProfileMatrix.keys()):
|
| 83 |
|
|
#print(docID)
|
| 84 |
|
|
legalSourceValues = line[legalSourcePosition]
|
| 85 |
|
|
legalSourceValues = legalSourceValues.split(";")
|
| 86 |
|
|
legalSourceValues = FieldFullValueDedupe(legalSourceValues)
|
| 87 |
|
|
|
| 88 |
|
|
downGradeSet = highProfileMatrix[docID]
|
| 89 |
|
|
remainderLegalSources = legalSourceValues - downGradeSet
|
| 90 |
|
|
attnyStillExists = False
|
| 91 |
|
|
for remainValue in remainderLegalSources:
|
| 92 |
|
|
if "*" in remainValue:
|
| 93 |
|
|
attnyStillExists = True
|
| 94 |
|
|
|
| 95 |
|
|
if attnyStillExists == False:
|
| 96 |
|
|
outputFile.write(f"Once downgrades removed from {docID}, no attorney names will be left!!\n")
|
| 97 |
|
|
else:
|
| 98 |
|
|
outputFile.write(f"Even after removing {';'.join(downGradeSet)}, the following will still be in the legal sources for {docID}, {';'.join(remainderLegalSources)}.\n")
|
| 99 |
|
|
|
| 100 |
|
|
outputFile.close()
|
| 101 |
|
|
## print(f"There are {len(highProfileMatrix.keys())} doc IDs with high conf downgrades.")
|
| 102 |
|
|
## testSet = set()
|
| 103 |
|
|
## for k in list(highProfileMatrix.keys()):
|
| 104 |
|
|
## for i in highProfileMatrix[k]:
|
| 105 |
|
|
## testSet.add(i)
|
| 106 |
|
|
## print(f"There are {len(testSet)} unique names across all of these.")
|
| 107 |
|
|
## for i in testSet:
|
| 108 |
|
|
## print(i)
|
