ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/ns_dev/Python/NinoCode/Active_prgs/Redgrave/EndoIGServerAnalyzer.py
Revision: 791
Committed: Tue Sep 13 14:10:49 2022 UTC (3 years, 6 months ago) by nino.borges
Content type: text/x-python
File size: 21064 byte(s)
Log Message:
Updated to support the MS Assessment and Planning Toolkit SQL report that Balaji sent to me.  Also updated the Aspect SQL file section to further parse the server name since I found that they are mixing regular servernames with full servernames like foo.bar.com.

File Contents

# User Rev Content
1 nino.borges 771 """
2    
3     EndoIGServerAnalyzer
4    
5     Created by:
6     Emanuel Borges
7     07.28.2022
8    
9     This program will take two CSV files and do a row to row compare. For this to work both CSVs need to have the same colums.
10     Set colOmitList to FALSE if you dont want it to parse at all and remove any columns. for that to work though you have to normalize the date
11     and number fields.
12    
13     NOTE: the application List file HAS pipes in it, so carefully select which columns you use.
14    
15     """
16    
17     import os
18    
19    
20     if __name__ == '__main__':
21    
22 nino.borges 791 outputFileName = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\20220907-TEST-WindowsServer8_Matrix_Report.csv"
23 nino.borges 771
24     ## This should all be csvs with pre-selected fields that are verified didnt already contain pipes. Also a headder row.
25     #win2kServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\Win8Servers_filtered.csv"
26     win2kServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\Win8Servers_NOFilter.csv"
27    
28     appListServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\AppInventory_extraFields.csv"
29     sqlServersListFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\Endo SQL Servers List__As of July 2022.csv"
30     prevSqlServersListFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\SQL Server Report 07212021.csv"
31     aspectDatabasesToServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\All_endo_SQL_Server_databases.csv"
32 nino.borges 783 serviceNowToServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\ServiceNow-PreParsed.txt"
33 nino.borges 786 legacySnowToServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\LegacySNOW-PreParsed.txt"
34 nino.borges 789 win2kChngReqToServersFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\SN-FromWinFile-PreParsed.txt"
35 nino.borges 791 crawledSqlServersListFile = r"C:\Users\eborges\Documents\Cases\Endo\20220715 - IG Projects\4 - Develop Identification Process to Inventory NCDS stored in Endo IT\_fromEndo\_csv\SqlServerDatabaseDetails-08-17-2022-21h44m56s-DatabaseSummaryONY.csv"
36 nino.borges 771
37     win2kServersFileMatrix = {}
38     appListServersFileMatrix = {}
39     sqlServersListFileMatrix = {}
40     prevSqlServersListFileMatrix = {}
41     aspectDatabasesToServersMatrix = {}
42 nino.borges 783 serviceNowToServersMatrix = {}
43 nino.borges 786 legacySnowToServersMatrix = {}
44 nino.borges 789 win2kChngReqToServersMatrix = {}
45 nino.borges 791 crawledSqlServersMatrix = {}
46 nino.borges 786
47 nino.borges 771
48     ## Start with ingesting the win2kservers file but be sure to save the headders on this one.
49     contents = open(win2kServersFile).readlines()
50     win2kServersFileHeadders = contents[0]
51     win2kServersFileHeadders = win2kServersFileHeadders.replace("\n","")
52     contents = contents[1:]
53     for line in contents:
54     line = line.replace("\n","")
55     parsedLine = line.split("|")
56     if parsedLine[0].upper() in list(win2kServersFileMatrix.keys()):
57     win2kServersFileMatrix[parsedLine[0].upper()].append(line)
58     else:
59     win2kServersFileMatrix[parsedLine[0].upper()] = [line,]
60     print("There are %s servers to start with."%len(list(win2kServersFileMatrix.keys())))
61    
62    
63     ## Next ingest the SQL servers list
64     contents = open(sqlServersListFile).readlines()
65     contents = contents[1:]
66     for line in contents:
67     line = line.replace("\n","")
68     parsedLine = line.split("|")
69     if parsedLine[0].upper() in list(sqlServersListFileMatrix.keys()):
70     sqlServersListFileMatrix[parsedLine[0].upper()].append(line)
71     else:
72     sqlServersListFileMatrix[parsedLine[0].upper()] = [line,]
73     print("There are %s servers on the SQL Servers file."%len(list(sqlServersListFileMatrix.keys())))
74    
75    
76     ## Next ingest the older SQL servers list
77     contents = open(prevSqlServersListFile).readlines()
78     contents = contents[1:]
79     for line in contents:
80     line = line.replace("\n","")
81     parsedLine = line.split("|")
82     if parsedLine[0].upper() in list(prevSqlServersListFileMatrix.keys()):
83     prevSqlServersListFileMatrix[parsedLine[0].upper()].append(line)
84     else:
85     prevSqlServersListFileMatrix[parsedLine[0].upper()] = [line,]
86     print("There are %s servers on the older SQL servers file."%len(list(prevSqlServersListFileMatrix.keys())))
87    
88    
89     ## Next ingest the databasename to server list from aspect
90     contents = open(aspectDatabasesToServersFile).readlines()
91     contents = contents[1:]
92     for line in contents:
93     line = line.replace("\n","")
94     parsedLine = line.split("|")
95 nino.borges 791 sName = parsedLine[0]
96     sName = sName.split(".")[0]
97     if sName.upper() in list(aspectDatabasesToServersMatrix.keys()):
98     aspectDatabasesToServersMatrix[sName.upper()].append(line)
99 nino.borges 771 else:
100 nino.borges 791 aspectDatabasesToServersMatrix[sName.upper()] = [line,]
101    
102     ## if parsedLine[0].upper() in list(aspectDatabasesToServersMatrix.keys()):
103     ## aspectDatabasesToServersMatrix[parsedLine[0].upper()].append(line)
104     ## else:
105     ## aspectDatabasesToServersMatrix[parsedLine[0].upper()] = [line,]
106 nino.borges 771 print("There are %s servers on the aspect databases to servers file."%len(list(aspectDatabasesToServersMatrix.keys())))
107    
108     ## Next is the app inventory which must first be parsed.
109     contents = open(appListServersFile).readlines()
110     contents = contents[1:]
111     for line in contents:
112     line = line.replace("\n","")
113     parsedLine = line.split("|")
114     parsedServerList = parsedLine[4].replace("#","")
115     parsedServerList = parsedServerList.split(";")
116     for s in parsedServerList:
117     if s.isnumeric():
118     pass
119     else:
120     if s.upper() in list(appListServersFileMatrix.keys()):
121     appListServersFileMatrix[s.upper()].append(line)
122     else:
123     appListServersFileMatrix[s.upper()] = [line,]
124     print("There are %s servers on the application list."%len(list(appListServersFileMatrix.keys())))
125    
126 nino.borges 783 ## Next ingest the ServiceNow Export to server list. check that there is a chg number at all.
127     contents = open(serviceNowToServersFile).readlines()
128     contents = contents[1:]
129     for line in contents:
130     line = line.replace("\n","")
131     parsedLine = line.split("|")
132     ## Dont add it to the matrix if there is no chg number.
133     if parsedLine[1]:
134     if parsedLine[0].upper() in list(serviceNowToServersMatrix.keys()):
135     serviceNowToServersMatrix[parsedLine[0].upper()].append(line)
136     else:
137     serviceNowToServersMatrix[parsedLine[0].upper()] = [line,]
138     print("There are %s servers on the ServiceNow Export to servers file."%len(list(serviceNowToServersMatrix.keys())))
139 nino.borges 771
140 nino.borges 787 ## Next ingest the Legacy SNOW Export to server list. check that there is a chg number at all.
141     contents = open(legacySnowToServersFile).readlines()
142     contents = contents[1:]
143     for line in contents:
144     line = line.replace("\n","")
145     parsedLine = line.split("|")
146     ## Dont add it to the matrix if there is no chg number.
147     if parsedLine[1]:
148     if parsedLine[0].upper() in list(legacySnowToServersMatrix.keys()):
149     legacySnowToServersMatrix[parsedLine[0].upper()].append(line)
150     else:
151     legacySnowToServersMatrix[parsedLine[0].upper()] = [line,]
152     print("There are %s servers on the Legacy SNOW Export to servers file."%len(list(legacySnowToServersMatrix.keys())))
153    
154    
155 nino.borges 789 ## Next ingest the SN Numbers from the original win2008 file Export to server list. check that there is a chg number at all.
156     contents = open(win2kChngReqToServersFile).readlines()
157     contents = contents[1:]
158     for line in contents:
159     line = line.replace("\n","")
160     parsedLine = line.split("|")
161     ## Dont add it to the matrix if there is no chg number.
162     if parsedLine[1]:
163     if parsedLine[0].upper() in list(win2kChngReqToServersMatrix.keys()):
164     win2kChngReqToServersMatrix[parsedLine[0].upper()].append(line)
165     else:
166     win2kChngReqToServersMatrix[parsedLine[0].upper()] = [line,]
167     print("There are %s servers on the SN Numbers from the original win2008 file Export to servers file."%len(list(win2kChngReqToServersMatrix.keys())))
168    
169    
170 nino.borges 791 ## Next ingest the crawled SQL databasename to server list
171     contents = open(crawledSqlServersListFile).readlines()
172     contents = contents[1:]
173     for line in contents:
174     line = line.replace("\n","")
175     parsedLine = line.split("|")
176     if parsedLine[0].upper() in list(crawledSqlServersMatrix.keys()):
177     crawledSqlServersMatrix[parsedLine[0].upper()].append(line)
178     else:
179     crawledSqlServersMatrix[parsedLine[0].upper()] = [line,]
180     print("There are %s servers on the crawled SQL databases to servers file."%len(list(crawledSqlServersMatrix.keys())))
181 nino.borges 789
182 nino.borges 791
183    
184    
185 nino.borges 771 outputFile = open(outputFileName,'w')
186 nino.borges 791 outputFile.write(win2kServersFileHeadders + "|Found on 2022 SQL Server Report?|Found on 2021 SQL Server Report?|Found DatabaseNames Server Report?|DB Names Found on AspectList|Found on Application Inventory?|Applications|App IT Leadership Owner|App IT System Owner|App Executive Leadership Owner|App Business Owner|Found on ServiceNow|ServiceNowChangeReqNumbers|ServiceNow Business Owner|ServiceNowChangeBusiness/IT Approver|ServiceNow Parent Change Req Numbers|Found on LegacySNOW|Legacy SNOW ChangeRequestNumbers|Win2008 Report ChngRequest Found|Win2008 Report ChngRequestNumbers|Win2008 Report SN Business Owner|Win2008 Report SN Business/IT Approver|Win2008 Report SN Parent Change Req Numbers|Found Crawled DatabaseNames Server Report?|DB Names Found on CrawledSQLList\n")
187 nino.borges 771 for i in list(win2kServersFileMatrix.keys()):
188     nl=outputFile.write(win2kServersFileMatrix[i][0])
189    
190     if i in list(sqlServersListFileMatrix.keys()):
191     nl=outputFile.write("|YES")
192     else:
193     nl=outputFile.write("|NO")
194     if i in list(prevSqlServersListFileMatrix.keys()):
195     nl=outputFile.write("|YES")
196     else:
197     nl=outputFile.write("|NO")
198     dbNamesList = ""
199     if i in list(aspectDatabasesToServersMatrix.keys()):
200     nl=outputFile.write("|YES")
201     for dbNameLine in aspectDatabasesToServersMatrix[i]:
202     dbNameLine = dbNameLine.replace("\n","")
203     dbNameLine = dbNameLine.split("|")
204     if dbNamesList:
205     dbNamesList = dbNamesList + "; " +dbNameLine[1]
206     else:
207     dbNamesList = dbNameLine[1]
208     else:
209     nl=outputFile.write("|NO")
210     if dbNamesList:
211     nl=outputFile.write("|" + dbNamesList)
212     else:
213     nl=outputFile.write("|")
214 nino.borges 772 appList = set()
215     itLeadershipOwnerList = set()
216     itSystemOwnerList = set()
217     executiveLeadershipOwnerList = set()
218     businessOwnerList = set()
219     tempOwnerList = ""
220 nino.borges 771 if i in list(appListServersFileMatrix.keys()):
221     nl=outputFile.write("|YES")
222     for appLine in appListServersFileMatrix[i]:
223     appLine = appLine.replace("\n","")
224     appLine = appLine.split("|")
225 nino.borges 772 appList.add(appLine[1]+' [AppID=%s]'%appLine[0])
226     tempOwnerList = appLine[5].replace("#","")
227     tempOwnerList = tempOwnerList.split(";")
228     for o in tempOwnerList:
229     if o.isnumeric():
230     pass
231     else:
232     itLeadershipOwnerList.add(o+' [AppID=%s]'%appLine[0])
233     tempOwnerList = appLine[6].replace("#","")
234     tempOwnerList = tempOwnerList.split(";")
235     for o in tempOwnerList:
236     if o.isnumeric():
237     pass
238     else:
239     itSystemOwnerList.add(o+' [AppID=%s]'%appLine[0])
240     tempOwnerList = appLine[7].replace("#","")
241     tempOwnerList = tempOwnerList.split(";")
242     for o in tempOwnerList:
243     if o.isnumeric():
244     pass
245     else:
246     executiveLeadershipOwnerList.add(o+' [AppID=%s]'%appLine[0])
247     tempOwnerList = appLine[8].replace("#","")
248     tempOwnerList = tempOwnerList.split(";")
249     for o in tempOwnerList:
250     if o.isnumeric():
251     pass
252     else:
253     businessOwnerList.add(o+' [AppID=%s]'%appLine[0])
254     ## if appList:
255     ## appList = appList + '; ' + appLine[1]+' [AppID=%s]'%appLine[0]
256     ## else:
257     ## appList = appLine[1]+' [AppID=%s]'%appLine[0]
258 nino.borges 771 else:
259     nl=outputFile.write("|NO")
260     if appList:
261 nino.borges 772 nl=outputFile.write("|" + "; ".join(appList))
262 nino.borges 771 else:
263     nl=outputFile.write("|")
264 nino.borges 772 if itLeadershipOwnerList:
265     nl=outputFile.write("|" + "; ".join(itLeadershipOwnerList))
266     else:
267     nl=outputFile.write("|")
268     if itSystemOwnerList:
269     nl=outputFile.write("|" + "; ".join(itSystemOwnerList))
270     else:
271     nl=outputFile.write("|")
272     if executiveLeadershipOwnerList:
273     nl=outputFile.write("|" + "; ".join(executiveLeadershipOwnerList))
274     else:
275     nl=outputFile.write("|")
276     if businessOwnerList:
277     nl=outputFile.write("|" + "; ".join(businessOwnerList))
278     else:
279     nl=outputFile.write("|")
280 nino.borges 783
281    
282    
283     chngReqList = ""
284 nino.borges 784 snBusinessOwnerList = set()
285 nino.borges 785 snApproverOwnerList = set()
286 nino.borges 786 snParentChgReqNumbList = set()
287    
288 nino.borges 783 if i in list(serviceNowToServersMatrix.keys()):
289     nl=outputFile.write("|YES")
290     for chngReqLine in serviceNowToServersMatrix[i]:
291     chngReqLine = chngReqLine.replace("\n","")
292     chngReqLine = chngReqLine.split("|")
293     chngReqDt = chngReqLine[2]
294     ## Grab just the date for now but can change to date and time if needed
295     chngReqDt = chngReqDt.split(" ")[0]
296     chngReqNumbWDate = "%s [%s]"%(chngReqLine[1],chngReqDt)
297     if chngReqList:
298     chngReqList = chngReqList + "; " +chngReqNumbWDate
299     else:
300     chngReqList = chngReqNumbWDate
301 nino.borges 784 if chngReqLine[3]:
302 nino.borges 785 if chngReqLine[3] == "None":
303     pass
304     else:
305     snBusinessOwnerList.add("%s [%s]"%(chngReqLine[3],chngReqLine[1]))
306     if chngReqLine[4]:
307     if chngReqLine[4] == "None":
308     pass
309     else:
310     snApproverOwnerList.add("%s [%s]"%(chngReqLine[4],chngReqLine[1]))
311 nino.borges 786 if chngReqLine[5]:
312     if chngReqLine[5] == "None":
313     pass
314     else:
315     snParentChgReqNumbList.add("%s [%s]"%(chngReqLine[5],chngReqLine[1]))
316 nino.borges 784
317 nino.borges 783 else:
318     nl=outputFile.write("|NO")
319     if chngReqList:
320     nl=outputFile.write("|" + chngReqList)
321     else:
322     nl=outputFile.write("|")
323 nino.borges 784 if snBusinessOwnerList:
324     nl=outputFile.write("|" + "; ".join(snBusinessOwnerList))
325     else:
326     nl=outputFile.write("|")
327 nino.borges 785 if snApproverOwnerList:
328     nl=outputFile.write("|" + "; ".join(snApproverOwnerList))
329     else:
330     nl=outputFile.write("|")
331 nino.borges 786 if snParentChgReqNumbList:
332     nl=outputFile.write("|" + "; ".join(snParentChgReqNumbList))
333     else:
334     nl=outputFile.write("|")
335 nino.borges 783
336    
337 nino.borges 787 snowChngReqList = ""
338     if i in list(legacySnowToServersMatrix.keys()):
339     nl=outputFile.write("|YES")
340     for snowChngReqLine in legacySnowToServersMatrix[i]:
341     snowChngReqLine = snowChngReqLine.replace("\n","")
342     snowChngReqLine = snowChngReqLine.split("|")
343     snowChngReqDt = snowChngReqLine[2]
344     ## Grab just the date for now but can change to date and time if needed
345     snowChngReqDt = snowChngReqDt.split(" ")[0]
346     snowChngReqNumbWDate = "%s [%s]"%(snowChngReqLine[1],snowChngReqDt)
347     if snowChngReqList:
348     snowChngReqList = snowChngReqList + "; " +snowChngReqNumbWDate
349     else:
350     snowChngReqList = snowChngReqNumbWDate
351     else:
352     nl=outputFile.write("|NO")
353    
354     if snowChngReqList:
355     nl=outputFile.write("|" + snowChngReqList)
356     else:
357     nl=outputFile.write("|")
358    
359    
360 nino.borges 789
361    
362     chngReqList = ""
363     snBusinessOwnerList = set()
364     snApproverOwnerList = set()
365     snParentChgReqNumbList = set()
366    
367     if i in list(win2kChngReqToServersMatrix.keys()):
368     nl=outputFile.write("|YES")
369     for chngReqLine in win2kChngReqToServersMatrix[i]:
370     chngReqLine = chngReqLine.replace("\n","")
371     chngReqLine = chngReqLine.split("|")
372     chngReqDt = chngReqLine[3]
373     ## Grab just the date for now but can change to date and time if needed
374     chngReqDt = chngReqDt.split(" ")[0]
375     chngReqNumbWDate = "%s [%s]"%(chngReqLine[1],chngReqDt)
376     if chngReqList:
377     chngReqList = chngReqList + "; " +chngReqNumbWDate
378     else:
379     chngReqList = chngReqNumbWDate
380     if chngReqLine[4]:
381     if chngReqLine[4] == "None":
382     pass
383     else:
384     snBusinessOwnerList.add("%s [%s]"%(chngReqLine[4],chngReqLine[1]))
385     if chngReqLine[5]:
386     if chngReqLine[5] == "None":
387     pass
388     else:
389     snApproverOwnerList.add("%s [%s]"%(chngReqLine[5],chngReqLine[1]))
390     if chngReqLine[6]:
391     if chngReqLine[6] == "None":
392     pass
393     else:
394     snParentChgReqNumbList.add("%s [%s]"%(chngReqLine[6],chngReqLine[1]))
395    
396     else:
397     nl=outputFile.write("|NO")
398     if chngReqList:
399     nl=outputFile.write("|" + chngReqList)
400     else:
401     nl=outputFile.write("|")
402     if snBusinessOwnerList:
403     nl=outputFile.write("|" + "; ".join(snBusinessOwnerList))
404     else:
405     nl=outputFile.write("|")
406     if snApproverOwnerList:
407     nl=outputFile.write("|" + "; ".join(snApproverOwnerList))
408     else:
409     nl=outputFile.write("|")
410     if snParentChgReqNumbList:
411     nl=outputFile.write("|" + "; ".join(snParentChgReqNumbList))
412     else:
413     nl=outputFile.write("|")
414    
415    
416 nino.borges 791
417     crawledDbNamesList = ""
418     if i in list(crawledSqlServersMatrix.keys()):
419     nl=outputFile.write("|YES")
420     for dbNameLine in crawledSqlServersMatrix[i]:
421     dbNameLine = dbNameLine.replace("\n","")
422     dbNameLine = dbNameLine.split("|")
423     if crawledDbNamesList:
424     crawledDbNamesList = crawledDbNamesList + "; " +dbNameLine[1]
425     else:
426     crawledDbNamesList = dbNameLine[1]
427     else:
428     nl=outputFile.write("|NO")
429     if crawledDbNamesList:
430     nl=outputFile.write("|" + crawledDbNamesList)
431     else:
432     nl=outputFile.write("|")
433    
434    
435 nino.borges 771 nl = outputFile.write("\n")
436     outputFile.close()