Active_prgs/Redgrave/Amazon_PerformDeepNamesNormQC.py

"""

Amazon_PerformDeepNamesNormQC

Created by:
Emanuel Borges
12.11.2024

This program is similar to Amazon_PerformNamesNormQC but it will perform a deeper level of names norm QC. I may just replace Amazon_PerformNamesNormQC with this file but for now i'd
like to keep both.

"""

import os, re
from uuid import UUID
import MyCode.Active_prgs.Redgrave.Amazon_PrivLogQC
import MyCode.Active_prgs.Redgrave.Amazon_NamesNormQC

version = '0.4.0'

issuesMatrix = {}

def GatherAllPossibleVariations(personMatch):
    """Takes a personMatch, which is the results of a person match, and attempts to make all possible name match variations that may exist in the formatted field.
    returns deduplicated list of tuple pairs (fullname, parenthetical)"""
    ##  Start as a plain list of all possible tuple pairs.
    allPossibleVariationsList = []

    allDomainsList = []
    if personMatch.work_email_address:
        allDomainsList.append(f"{personMatch.work_email_address.split('@')[-1]}")
    if personMatch.alt_work_email_address:
        allDomainsList.append(f"{personMatch.alt_work_email_address.split('@')[-1]}")
    allDomainsList = list(dict.fromkeys(allDomainsList))

    if personMatch.full_name_overide:
        fullName = personMatch.full_name_overide
        for domain in allDomainsList:
            allPossibleVariationsList.append((fullName,domain))
    if personMatch.full_name_preferred:
        ##  Going to need to do a bit of replacing to remove some information that is just never in the formatted.
        fullPreferredName = personMatch.full_name_preferred
        fullPreferredName = fullPreferredName.replace('(LEGAL)','')
        fullPreferredName = fullPreferredName.replace('(SHE, HER)','')
        fullPreferredName = fullPreferredName.replace('(SHE HER)','')
        if "," in fullPreferredName:
            preferedLastName, preferedFirstName = fullPreferredName.split(',')
            preferedLastName = preferedLastName.strip()
            preferedFirstName = preferedFirstName.strip()
            preferedFirstName = preferedFirstName.split(" ")[0]
            fullName = f"{preferedFirstName} {preferedLastName}"
            #fullName = f"{preferedLastName}, {preferedFirstName}"
            for domain in allDomainsList:
                allPossibleVariationsList.append((fullName,domain))
        else:
            print(f"ERROR in this name {fullPreferredName}")
    if personMatch.last_name:
        if personMatch.first_name:
            fullName = f"{personMatch.first_name} {personMatch.last_name}"
            #fullName = f"{personMatch.last_name}, {personMatch.first_name}"
        else:
            fullName = f"{personMatch.last_name}"
        for domain in allDomainsList:
            allPossibleVariationsList.append((fullName,domain))


    ##  Now return a deduplicated list by using dict to deduplicate.
    return list(dict.fromkeys(allPossibleVariationsList))


def AddToIssuesList(docID,issueMessage):
    """This function will add a single issue to the issues matrix."""
    if docID in list(issuesMatrix.keys()):
        issuesMatrix[docID].append(issueMessage)
    else:
        issuesMatrix[docID] = [issueMessage,]
    

if __name__ == '__main__':
    cleanedDatExportFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Testing_3\PrivLogExport_20241204_VEAS_Converted.txt"
    masterAttorneyListFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Amazon_ Master Attorney List  2024.12.12(20241212-1151).xlsx"
    fullNameOveridesFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Consilio\VEAS-MasterAttorneyList\FullNameOverides.txt"
    outputFileName = r"C:\Test_Dir\Amazon\NameNormDeepOutputText.txt"


    nv = MyCode.Active_prgs.Redgrave.Amazon_NamesNormQC.NamesVerification(cleanedDatExportFileName, masterAttorneyListFileName, fullNameOveridesFileName)

    qcP = MyCode.Active_prgs.Redgrave.Amazon_PrivLogQC.QcPrivLog(cleanedDatExportFileName)

    #issuesMatrix = {}

    print(f"\nThere are {len(qcP.formattedValuesDict)} documents in the formatted values dictionary.")
    print(f"There are {len(qcP.metadataValuesDict)} documents in the metadata values dictionary.")

    workList = qcP.metadataValuesDict.keys()
    for docID in workList:
        metadataFieldValues = qcP.metadataValuesDict[docID]._asdict()['toValues']
        formattedFieldValues = qcP.formattedValuesDict[docID]._asdict()['toValues']
        ##  remember to convert all values in formattedFieldValues to uppercase (perhaps eventually do some of the formatted cleaning that eli mentioned.
        formattedFieldValues = [xVal.upper() for xVal in formattedFieldValues]
        ##  This will change once you start itterating acroll all of the field values names
        currentMetadataValues = metadataFieldValues
        for val in currentMetadataValues:
            ##  First try to locate an email address in this val and if found, try to find that in the MAL.
            results = re.findall(qcP.allPossibleEmailAddressesRegExPattern, val)
            if results:
                ##  Use some smart deduplication to remove duplicates.
                results = nv.SmartDedupeSet(results)
                for result in results:
                    ##  Try to find a match in the MAL by email. There shouldnt rows with duplicative email addresses.
                    ##  TODO:DONE: Update search_by_email to search both workemail and alt email.
                    
                    personMatch = nv.malPeopleList.search_by_email(result.upper())
                    if personMatch:
                        ##  Person match found in MAL.  Now try to match a value in the formatted field by pulling various values from the MAL.
                        ##  For each of these match attempts, try using the correct designation and incorrect designation (* vs no *) and note that.
                        allPossibleVariationsList = GatherAllPossibleVariations(personMatch)
                        matchFlag = False
                        for variationPair in allPossibleVariationsList:
                            if personMatch.is_attorney == 'YES':
                                if f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
                                    ##  This variation was found in the list of formatted values, which is fine, so just remove it.
                                    if matchFlag:
                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
                                    formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
                                    matchFlag = True
                                elif f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
                                    ##  This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
                                    if matchFlag:
                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
                                    formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
                                    matchFlag = True
                                    ##  TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
                                    AddToIssuesList(docID,f"{variationPair[0]} ({variationPair[1]}) in To Field is High Confidence Potential Upgrade")

                            else:
                                ##  TODO: will need to split this out to include split role soon.
                                if f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
                                    if matchFlag:
                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
                                    ##  This variation was found in the list of formatted values, which is fine, so just remove it.
                                    formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
                                    matchFlag = True
                                elif f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
                                    ##  This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
                                    if matchFlag:
                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
                                    formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
                                    matchFlag = True
                                    ##  TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
                                    AddToIssuesList(docID,f"{variationPair[0]}* ({variationPair[1]}) in To Field is High Confidence Potential downgrade")
                                
                    else:
                        ##  Person match, using email, not found in MAL.  Try extracting a name from this metadata value and try matching the MAL using that.
                        val = val.upper()
                        ##  First lets try dealing with the extra parentheticals that keep coming up but make sure to handle (LEGAL) differently.
                        if "(LEGAL)" in val:
                            ##  Attempt to only remove the email parenthetical, including the now empty paren.
                            val = val.replace(result.upper(),"")
                            val = val.replace("()",'')
                            #val = val.replace(")","")
                        else:
                            ##  Remove all parenthicals, including any character in that paren, from value.
                            val = re.sub(r"\([^)]*\)","",val)
                        val = val.strip()
                        ##  with the email address and the paren stripped out of the val, only move forward if anything still exists.
                        if val:
                            ##  if there is a comma, parse to last name, first name
                            if "," in val:
                                lastName, firstName = val.split(",")
                                lastName = lastName.strip()
                                firstName = firstName.strip()
                            elif " " in val:
                                ##  For now, try just splitting by the first space and take everything after as the first name.
                                firstName, lastName = val.split(" ",1)
                            ##  With the name now parse, try searching for all values that match on the last name.
                            
                            personMatchList = nv.malPeopleList.return_list_of_matching_values('last_name',lastName)
                            if personMatchList:
                                possiblePeopleMatchesMatrix = {}
                                ##  For each personMatch in the list, now attempt to also see if the first name matches and, if so, put that into the possible people list
                                for personMatch in personMatchList:
                                    if personMatch.first_name == firstName:
                                        ##  This is a personMatch that matches the first and last name
                                        possiblePeopleMatchesMatrix[personMatch._id] = 1
                                if possiblePeopleMatchesMatrix.keys():
                                    ##  If the list of possible matches is just 1, we are okay doing a simple match attempt.  if more than 1, we need to test for conflicting designations in the list of possible matches.
                                    if len(list(possiblePeopleMatchesMatrix.keys())) < 2:
                                        allPossibleVariationsList = GatherAllPossibleVariations(personMatch)
                                        matchFlag = False
                                        for variationPair in allPossibleVariationsList:
                                            if personMatch.is_attorney == 'YES':
                                                if f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
                                                    ##  This variation was found in the list of formatted values, which is fine, so just remove it.
                                                    if matchFlag:
                                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE name VALUE??")
                                                    formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
                                                    matchFlag = True
                                                elif f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
                                                    ##  This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
                                                    if matchFlag:
                                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE name VALUE??")
                                                    formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
                                                    matchFlag = True
                                                    ##  TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
                                                    AddToIssuesList(docID,f"{variationPair[0]} ({variationPair[1]}) in To Field is Low Confidence Potential Upgrade")

                                            else:
                                                ##  TODO: will need to split this out to include split role soon.
                                                if f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
                                                    if matchFlag:
                                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
                                                    ##  This variation was found in the list of formatted values, which is fine, so just remove it.
                                                    formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
                                                    matchFlag = True
                                                elif f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
                                                    ##  This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
                                                    if matchFlag:
                                                        print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
                                                    formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
                                                    matchFlag = True
                                                    ##  TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
                                                    AddToIssuesList(docID,f"{variationPair[0]}* ({variationPair[1]}) in To Field is Low Confidence Potential downgrade")
                                    
                            
                            else:
                                ##  TODO: Need to ask Eli if I dont a match by checking first and last name for a match if it's needed to flag these.
                                #AddToIssuesList(docID,f"first name: {firstName} - last name: {lastName} is an email in metadata that I couldnt match in MAL")
                                pass

            else:
                ##  No email address could be extracted from this val. Try extracting a name from this metadata value and try matching the MAL using that.
                AddToIssuesList(docID,f"{val} is a value in metadata that I couldnt extract an email address from")

            
        ##  Since you itterated over the metadata values but didnt itterate over the formatted values, check for any remaining formatted values that exist in the list
        if formattedFieldValues:
            for val in formattedFieldValues:
                ##  TODO: Confirm with Eli but we should only report these remaining values if they have a *
                ##  From Eliu: the Highest risk is the * values because these are the potential overdesignations so yes but in a perfect world we would check both.
                if "*" in val:
                    ##  TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
                    AddToIssuesList(docID,f"{val} in To Field is an attorney but couldnt be matched to any metadata value.")


    ##  Now just unpack and write the issues, per DocID, to the output file separated by semicolon.
    outputFile = open(outputFileName,'w')
    for docID in list(issuesMatrix.keys()):
        outputFile.write(f"{docID}|{';'.join(issuesMatrix[docID])}\n")
    outputFile.close()
Revision:	857
Committed:	Fri Dec 13 16:33:58 2024 UTC (15 months, 1 week ago) by nino.borges
Content type:	text/x-python
File size:	18221 byte(s)
Log Message:	With the additions to the issues list now a function, I simply removed the commented out old lines. Again tested it to be sure I didnt mess something up.
#	User	Rev	Content
1	nino.borges	855	"""
2
3			Amazon_PerformDeepNamesNormQC
4
5			Created by:
6			Emanuel Borges
7			12.11.2024
8
9			This program is similar to Amazon_PerformNamesNormQC but it will perform a deeper level of names norm QC. I may just replace Amazon_PerformNamesNormQC with this file but for now i'd
10			like to keep both.
11
12			"""
13
14			import os, re
15			from uuid import UUID
16			import MyCode.Active_prgs.Redgrave.Amazon_PrivLogQC
17			import MyCode.Active_prgs.Redgrave.Amazon_NamesNormQC
18
19	nino.borges	857	version = '0.4.0'
20	nino.borges	855
21			issuesMatrix = {}
22
23			def GatherAllPossibleVariations(personMatch):
24			"""Takes a personMatch, which is the results of a person match, and attempts to make all possible name match variations that may exist in the formatted field.
25			returns deduplicated list of tuple pairs (fullname, parenthetical)"""
26			## Start as a plain list of all possible tuple pairs.
27			allPossibleVariationsList = []
28
29			allDomainsList = []
30			if personMatch.work_email_address:
31			allDomainsList.append(f"{personMatch.work_email_address.split('@')[-1]}")
32			if personMatch.alt_work_email_address:
33			allDomainsList.append(f"{personMatch.alt_work_email_address.split('@')[-1]}")
34			allDomainsList = list(dict.fromkeys(allDomainsList))
35
36			if personMatch.full_name_overide:
37			fullName = personMatch.full_name_overide
38			for domain in allDomainsList:
39			allPossibleVariationsList.append((fullName,domain))
40			if personMatch.full_name_preferred:
41			## Going to need to do a bit of replacing to remove some information that is just never in the formatted.
42			fullPreferredName = personMatch.full_name_preferred
43			fullPreferredName = fullPreferredName.replace('(LEGAL)','')
44			fullPreferredName = fullPreferredName.replace('(SHE, HER)','')
45			fullPreferredName = fullPreferredName.replace('(SHE HER)','')
46			if "," in fullPreferredName:
47			preferedLastName, preferedFirstName = fullPreferredName.split(',')
48			preferedLastName = preferedLastName.strip()
49			preferedFirstName = preferedFirstName.strip()
50			preferedFirstName = preferedFirstName.split(" ")[0]
51			fullName = f"{preferedFirstName} {preferedLastName}"
52			#fullName = f"{preferedLastName}, {preferedFirstName}"
53			for domain in allDomainsList:
54			allPossibleVariationsList.append((fullName,domain))
55			else:
56			print(f"ERROR in this name {fullPreferredName}")
57			if personMatch.last_name:
58			if personMatch.first_name:
59			fullName = f"{personMatch.first_name} {personMatch.last_name}"
60			#fullName = f"{personMatch.last_name}, {personMatch.first_name}"
61			else:
62			fullName = f"{personMatch.last_name}"
63			for domain in allDomainsList:
64			allPossibleVariationsList.append((fullName,domain))
65
66
67			## Now return a deduplicated list by using dict to deduplicate.
68			return list(dict.fromkeys(allPossibleVariationsList))
69
70
71	nino.borges	856	def AddToIssuesList(docID,issueMessage):
72			"""This function will add a single issue to the issues matrix."""
73			if docID in list(issuesMatrix.keys()):
74			issuesMatrix[docID].append(issueMessage)
75			else:
76			issuesMatrix[docID] = [issueMessage,]
77	nino.borges	855
78
79
80			if __name__ == '__main__':
81			cleanedDatExportFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Testing_3\PrivLogExport_20241204_VEAS_Converted.txt"
82			masterAttorneyListFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Amazon_ Master Attorney List 2024.12.12(20241212-1151).xlsx"
83			fullNameOveridesFileName = r"C:\Users\eborges\OneDrive - Redgrave LLP\Documents\Cases\Amazon\_PrivLogQCProcess\Consilio\VEAS-MasterAttorneyList\FullNameOverides.txt"
84			outputFileName = r"C:\Test_Dir\Amazon\NameNormDeepOutputText.txt"
85
86
87			nv = MyCode.Active_prgs.Redgrave.Amazon_NamesNormQC.NamesVerification(cleanedDatExportFileName, masterAttorneyListFileName, fullNameOveridesFileName)
88
89			qcP = MyCode.Active_prgs.Redgrave.Amazon_PrivLogQC.QcPrivLog(cleanedDatExportFileName)
90
91			#issuesMatrix = {}
92
93			print(f"\nThere are {len(qcP.formattedValuesDict)} documents in the formatted values dictionary.")
94			print(f"There are {len(qcP.metadataValuesDict)} documents in the metadata values dictionary.")
95
96			workList = qcP.metadataValuesDict.keys()
97			for docID in workList:
98			metadataFieldValues = qcP.metadataValuesDict[docID]._asdict()['toValues']
99			formattedFieldValues = qcP.formattedValuesDict[docID]._asdict()['toValues']
100			## remember to convert all values in formattedFieldValues to uppercase (perhaps eventually do some of the formatted cleaning that eli mentioned.
101			formattedFieldValues = [xVal.upper() for xVal in formattedFieldValues]
102			## This will change once you start itterating acroll all of the field values names
103			currentMetadataValues = metadataFieldValues
104			for val in currentMetadataValues:
105			## First try to locate an email address in this val and if found, try to find that in the MAL.
106			results = re.findall(qcP.allPossibleEmailAddressesRegExPattern, val)
107			if results:
108			## Use some smart deduplication to remove duplicates.
109			results = nv.SmartDedupeSet(results)
110			for result in results:
111			## Try to find a match in the MAL by email. There shouldnt rows with duplicative email addresses.
112			## TODO:DONE: Update search_by_email to search both workemail and alt email.
113
114			personMatch = nv.malPeopleList.search_by_email(result.upper())
115			if personMatch:
116			## Person match found in MAL. Now try to match a value in the formatted field by pulling various values from the MAL.
117			## For each of these match attempts, try using the correct designation and incorrect designation (* vs no *) and note that.
118			allPossibleVariationsList = GatherAllPossibleVariations(personMatch)
119			matchFlag = False
120			for variationPair in allPossibleVariationsList:
121			if personMatch.is_attorney == 'YES':
122			if f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
123			## This variation was found in the list of formatted values, which is fine, so just remove it.
124			if matchFlag:
125			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
126			formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
127			matchFlag = True
128			elif f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
129			## This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
130			if matchFlag:
131			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
132			formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
133			matchFlag = True
134	nino.borges	856	## TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
135			AddToIssuesList(docID,f"{variationPair[0]} ({variationPair[1]}) in To Field is High Confidence Potential Upgrade")
136	nino.borges	857
137	nino.borges	855	else:
138			## TODO: will need to split this out to include split role soon.
139			if f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
140			if matchFlag:
141			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
142			## This variation was found in the list of formatted values, which is fine, so just remove it.
143			formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
144			matchFlag = True
145			elif f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
146			## This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
147			if matchFlag:
148			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
149			formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
150			matchFlag = True
151	nino.borges	856	## TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
152			AddToIssuesList(docID,f"{variationPair[0]}* ({variationPair[1]}) in To Field is High Confidence Potential downgrade")
153	nino.borges	855
154			else:
155			## Person match, using email, not found in MAL. Try extracting a name from this metadata value and try matching the MAL using that.
156			val = val.upper()
157			## First lets try dealing with the extra parentheticals that keep coming up but make sure to handle (LEGAL) differently.
158			if "(LEGAL)" in val:
159			## Attempt to only remove the email parenthetical, including the now empty paren.
160			val = val.replace(result.upper(),"")
161			val = val.replace("()",'')
162			#val = val.replace(")","")
163			else:
164			## Remove all parenthicals, including any character in that paren, from value.
165			val = re.sub(r"\([^)]*\)","",val)
166			val = val.strip()
167			## with the email address and the paren stripped out of the val, only move forward if anything still exists.
168			if val:
169			## if there is a comma, parse to last name, first name
170			if "," in val:
171			lastName, firstName = val.split(",")
172			lastName = lastName.strip()
173			firstName = firstName.strip()
174			elif " " in val:
175			## For now, try just splitting by the first space and take everything after as the first name.
176			firstName, lastName = val.split(" ",1)
177			## With the name now parse, try searching for all values that match on the last name.
178
179			personMatchList = nv.malPeopleList.return_list_of_matching_values('last_name',lastName)
180			if personMatchList:
181			possiblePeopleMatchesMatrix = {}
182			## For each personMatch in the list, now attempt to also see if the first name matches and, if so, put that into the possible people list
183			for personMatch in personMatchList:
184			if personMatch.first_name == firstName:
185			## This is a personMatch that matches the first and last name
186			possiblePeopleMatchesMatrix[personMatch._id] = 1
187			if possiblePeopleMatchesMatrix.keys():
188			## If the list of possible matches is just 1, we are okay doing a simple match attempt. if more than 1, we need to test for conflicting designations in the list of possible matches.
189			if len(list(possiblePeopleMatchesMatrix.keys())) < 2:
190			allPossibleVariationsList = GatherAllPossibleVariations(personMatch)
191			matchFlag = False
192			for variationPair in allPossibleVariationsList:
193			if personMatch.is_attorney == 'YES':
194			if f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
195			## This variation was found in the list of formatted values, which is fine, so just remove it.
196			if matchFlag:
197			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE name VALUE??")
198			formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
199			matchFlag = True
200			elif f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
201			## This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
202			if matchFlag:
203			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE name VALUE??")
204			formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
205			matchFlag = True
206	nino.borges	856	## TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
207			AddToIssuesList(docID,f"{variationPair[0]} ({variationPair[1]}) in To Field is Low Confidence Potential Upgrade")
208	nino.borges	857
209	nino.borges	855	else:
210			## TODO: will need to split this out to include split role soon.
211			if f"{variationPair[0]} ({variationPair[1]})" in formattedFieldValues:
212			if matchFlag:
213			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
214			## This variation was found in the list of formatted values, which is fine, so just remove it.
215			formattedFieldValues.remove(f"{variationPair[0]} ({variationPair[1]})")
216			matchFlag = True
217			elif f"{variationPair[0]}* ({variationPair[1]})" in formattedFieldValues:
218			## This variation was found in the list of formatted values, however it's a bad match, so remove it but also add this to the issuesList.
219			if matchFlag:
220			print("WARNING: TWO SEPARATE FORMATTED NAME MATCHES FROM A SINGLE EMAIL VALUE??")
221			formattedFieldValues.remove(f"{variationPair[0]}* ({variationPair[1]})")
222			matchFlag = True
223	nino.borges	856	## TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
224			AddToIssuesList(docID,f"{variationPair[0]}* ({variationPair[1]}) in To Field is Low Confidence Potential downgrade")
225	nino.borges	855
226
227			else:
228			## TODO: Need to ask Eli if I dont a match by checking first and last name for a match if it's needed to flag these.
229	nino.borges	857	#AddToIssuesList(docID,f"first name: {firstName} - last name: {lastName} is an email in metadata that I couldnt match in MAL")
230	nino.borges	855	pass
231	nino.borges	857
232	nino.borges	855	else:
233			## No email address could be extracted from this val. Try extracting a name from this metadata value and try matching the MAL using that.
234	nino.borges	856	AddToIssuesList(docID,f"{val} is a value in metadata that I couldnt extract an email address from")
235	nino.borges	857
236	nino.borges	855
237			## Since you itterated over the metadata values but didnt itterate over the formatted values, check for any remaining formatted values that exist in the list
238			if formattedFieldValues:
239			for val in formattedFieldValues:
240			## TODO: Confirm with Eli but we should only report these remaining values if they have a *
241			## From Eliu: the Highest risk is the * values because these are the potential overdesignations so yes but in a perfect world we would check both.
242			if "*" in val:
243	nino.borges	856	## TODO: change the hard coded "To Field" here once you change to itterate over all field groups.
244			AddToIssuesList(docID,f"{val} in To Field is an attorney but couldnt be matched to any metadata value.")
245	nino.borges	855
246
247	nino.borges	857
248	nino.borges	855	## Now just unpack and write the issues, per DocID, to the output file separated by semicolon.
249			outputFile = open(outputFileName,'w')
250			for docID in list(issuesMatrix.keys()):
251			outputFile.write(f"{docID}\|{';'.join(issuesMatrix[docID])}\n")
252			outputFile.close()