Active_prgs/Gromulus/Gromulus_Lib.py

"""
This creates the main library of methods for Gromulus.

"""

import sqlite3, configparser, shutil
import xml.etree.ElementTree as ET
import Tool_Box.NinoGenTools, os


config = configparser.ConfigParser()
config.read("/home/nino/.gromulus/gromulus_cfg.ini")

conn = sqlite3.connect(config.get('RootConfig','databasePath')) #"GromulusDatabase.db")


def ImportNewNoIntroDat(datPath):
    """Deletes all of the data in the two No_Intro tables and load the info from the new dat."""
    tree = ET.parse(datPath)
    root = tree.getroot()

    ##  First gather information from header
    for elm in root.findall('header'):
        headerID = elm.find('id').text
        headerName = elm.find('name').text
        headerDesc = elm.find('description').text
        headerVer = elm.find('version').text
    
    #print(headerID)
    #print (headerName)
    #print(headerDesc)
    #print(headerVer)
    conn.execute("""DELETE FROM no_intro_system""" )
    conn.execute(f'INSERT INTO no_intro_system VALUES ({headerID}, "{headerName}", "{headerDesc}", "{headerVer}")')
    
    conn.execute("""DELETE FROM no_intro_main""" )
    for elm in root.findall('game'):
        gameName = elm.get('name')
        gameID = elm.get('id')
        gameCloneOfID = elm.get('cloneofid')
        gameDesc = elm.find('description').text

        romName = elm.find('rom').get('name')
        romSize = elm.find('rom').get('size')
        romCRC = elm.find('rom').get('crc')
        romMD5 = elm.find('rom').get('md5')
        romSHA1 = elm.find('rom').get('sha1')
        romSHA256 = elm.find('rom').get('sha256')
        romStatus = elm.find('rom').get('status')
        conn.execute(f'INSERT INTO no_intro_main VALUES ("{gameName}", "{gameID}", "{gameCloneOfID}", "{gameDesc}", "{romName}", "{romCRC}", "{romMD5}", "{romSHA1}", "{romSHA256}", "{romStatus}", {headerID})')

    conn.commit()
    conn.close()


def ImportNewTosecDat(datPath):
    """Imports a new TOSEC DAT file and populates the appropriate tables."""
    tree = ET.parse(datPath)
    root = tree.getroot()

    for elm in root.findall('header'):
        system_name = elm.find('name').text
        system_desc = elm.find('description').text

    conn.execute("DELETE FROM tosec_system")
    conn.execute("INSERT INTO tosec_system (system_name, system_description) VALUES (?, ?)",
                 (system_name, system_desc))
    system_id = conn.execute("SELECT id FROM tosec_system WHERE system_name = ?", (system_name,)).fetchone()[0]

    conn.execute("DELETE FROM tosec_main")
    for elm in root.findall('game'):
        game_name = elm.get('name')
        game_desc = elm.find('description').text
        rom = elm.find('rom')
        rom_name = rom.get('name')
        rom_crc = rom.get('crc')
        rom_md5 = rom.get('md5')
        rom_sha1 = rom.get('sha1')
        conn.execute("""INSERT INTO tosec_main 
                        (game_name, description, rom_name, crc, md5, sha1, system_id)
                        VALUES (?, ?, ?, ?, ?, ?, ?)""",
                     (game_name, game_desc, rom_name, rom_crc, rom_md5, rom_sha1, system_id))

    conn.commit()
    conn.close()


def AddNewSystem(systemName, shortName, relativePath):
    """Adding a single system, with path where the ROMs will live."""
    conn.execute(f'INSERT INTO main_app_system("name", "short_name", "relative_file_path")VALUES ("{systemName}", "{shortName}", "{relativePath}")')
    conn.commit()
    conn.close()

    ##  Next gather game information
def AddNewRoms(pathToRoms, systemID, testOnly = False):
    """Adding additional roms."""
    importedCount = 0
    alreadyExistsCount = 0
    errImportCount = 0

    sysResult = conn.execute(f'SELECT relative_file_path FROM main_app_system WHERE id={systemID}')
    for s in sysResult:
        systemRomPath = s[0]
    fullRomPath = os.path.join(config.get('RootConfig','softwareBackupStartDir'),systemRomPath)
    
    res = conn.execute('SELECT md5 FROM main_app')
    currentHashList = []
    for h in res:
        #print(h[0])
        currentHashList.append(h[0])
    #print(currentHashList)
    print(len(currentHashList))

    hshr = Tool_Box.NinoGenTools.HashFileContents('md5')
    for testFile in os.listdir(pathToRoms):
        print(f"\n\nNow analyzing {testFile}...")
        hashVal = hshr.HashFile(os.path.join(pathToRoms,testFile))
        if hashVal in currentHashList:
            print("This file is already in your collection, skipping.")
            alreadyExistsCount +=1
        else:
            print("This file is unique.  Adding to collection.")
            fileNameInc = 0
            targetFileName = testFile
            while os.path.isfile(os.path.join(fullRomPath,targetFileName)):
                fileNameInc +=1
                testFileNamePart, testFileExt = os.path.splitext(testFile)
                targetFileName = testFileNamePart +"_"+ str(fileNameInc) + testFileExt
            if testOnly:
                pass
            else:
                shutil.copyfile(os.path.join(pathToRoms,testFile),os.path.join(fullRomPath,targetFileName))
                conn.execute(f'INSERT INTO main_app("file_name", "system_console", "md5")VALUES("{testFile}","{systemID}","{hashVal}")')
                conn.commit()
            importedCount +=1
    conn.close()
    return importedCount, alreadyExistsCount, errImportCount

def GetSystemList():
    res = conn.execute("SELECT id, name, short_name, relative_file_path FROM main_app_system ")
    systemNamesMatrix = {}
    for h in res:
        systemNamesMatrix[h[1]] = [h[0],h[2],h[3]]
    return systemNamesMatrix

def GetGameListBySystem(systemName):
    res = conn.execute("SELECT main_app.id, main_app.game_name, no_intro_main.game_name, tosec_main.game_name,main_app.file_name FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5 LEFT JOIN tosec_main ON main_app.md5 = tosec_main.md5")
    #res = conn.execute("SELECT main_app.id, main_app.game_name, no_intro_main.game_name, main_app.file_name FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
    gameNamesMatrix = {}
    for h in res:
        if h[1]:
            gameNamesMatrix[h[1]] = h[0]
        elif h[2]:
            gameNamesMatrix[h[2]] = h[0]
        elif h[3]:
            gameNamesMatrix[h[3]] = h[0]
        else:
            gameNamesMatrix[h[4]] = h[0]

    #for g in gameNamesList:
    #    print(g)
    return gameNamesMatrix

def GetSingleGameById(id):
    res = conn.execute(f"""SELECT main_app.game_name, main_app.md5, main_app.file_name, main_app.relative_file_path, no_intro_main.game_name, no_intro_system.name FROM main_app 
    LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5 
    LEFT JOIN no_intro_system ON no_intro_main.no_intro_system_id = no_intro_system.id 
    WHERE main_app.id = {id}""")
    for h in res:
        gameTuple = h
        print(h)
    return gameTuple
    

if __name__ == '__main__':
    pass
    #test = GetSystemList()
    #print(test.keys())
    #AddNewSystem("Super Nintendo", "SNES", r"Roms/Snes/Gromulus")
    #ImportNewNoIntroDat("/home/nino/MyCode/Python/Active_prgs/Gromulus/_data/_dats/Nintendo - Super Nintendo Entertainment System (20230516-033614).dat")
    #AddNewRoms('/mnt/smb/HomeData/ninoborges/Emulation/Roms/Snes/Random2',"1",testOnly=True)
    #res = conn.execute("SELECT * FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
    #for h in res:
    #    print(h)
    #    print("\n")
    #GetGameListBySystem('snes')
    #GetSingleGameById('224')

Revision:	906
Committed:	Fri Jun 6 22:23:54 2025 UTC (9 months, 3 weeks ago) by nino.borges
Content type:	text/x-python
File size:	7534 byte(s)
Log Message:	Updated ImportNewNoIntroDat function to support a new schema where there is now a separate no intro system table.
#	User	Rev	Content
1	nino.borges	795	"""
2			This creates the main library of methods for Gromulus.
3
4			"""
5
6	nino.borges	805	import sqlite3, configparser, shutil
7	nino.borges	795	import xml.etree.ElementTree as ET
8			import Tool_Box.NinoGenTools, os
9
10
11	nino.borges	805	config = configparser.ConfigParser()
12			config.read("/home/nino/.gromulus/gromulus_cfg.ini")
13	nino.borges	795
14	nino.borges	805	conn = sqlite3.connect(config.get('RootConfig','databasePath')) #"GromulusDatabase.db")
15	nino.borges	795
16
17			def ImportNewNoIntroDat(datPath):
18			"""Deletes all of the data in the two No_Intro tables and load the info from the new dat."""
19			tree = ET.parse(datPath)
20			root = tree.getroot()
21
22			## First gather information from header
23			for elm in root.findall('header'):
24			headerID = elm.find('id').text
25			headerName = elm.find('name').text
26			headerDesc = elm.find('description').text
27			headerVer = elm.find('version').text
28
29			#print(headerID)
30			#print (headerName)
31			#print(headerDesc)
32			#print(headerVer)
33			conn.execute("""DELETE FROM no_intro_system""" )
34			conn.execute(f'INSERT INTO no_intro_system VALUES ({headerID}, "{headerName}", "{headerDesc}", "{headerVer}")')
35
36			conn.execute("""DELETE FROM no_intro_main""" )
37			for elm in root.findall('game'):
38			gameName = elm.get('name')
39			gameID = elm.get('id')
40			gameCloneOfID = elm.get('cloneofid')
41			gameDesc = elm.find('description').text
42
43			romName = elm.find('rom').get('name')
44			romSize = elm.find('rom').get('size')
45			romCRC = elm.find('rom').get('crc')
46			romMD5 = elm.find('rom').get('md5')
47			romSHA1 = elm.find('rom').get('sha1')
48			romSHA256 = elm.find('rom').get('sha256')
49			romStatus = elm.find('rom').get('status')
50			conn.execute(f'INSERT INTO no_intro_main VALUES ("{gameName}", "{gameID}", "{gameCloneOfID}", "{gameDesc}", "{romName}", "{romCRC}", "{romMD5}", "{romSHA1}", "{romSHA256}", "{romStatus}", {headerID})')
51
52			conn.commit()
53			conn.close()
54
55
56	nino.borges	906	def ImportNewTosecDat(datPath):
57			"""Imports a new TOSEC DAT file and populates the appropriate tables."""
58	nino.borges	806	tree = ET.parse(datPath)
59			root = tree.getroot()
60
61	nino.borges	906	for elm in root.findall('header'):
62			system_name = elm.find('name').text
63			system_desc = elm.find('description').text
64
65			conn.execute("DELETE FROM tosec_system")
66			conn.execute("INSERT INTO tosec_system (system_name, system_description) VALUES (?, ?)",
67			(system_name, system_desc))
68			system_id = conn.execute("SELECT id FROM tosec_system WHERE system_name = ?", (system_name,)).fetchone()[0]
69
70			conn.execute("DELETE FROM tosec_main")
71	nino.borges	806	for elm in root.findall('game'):
72	nino.borges	906	game_name = elm.get('name')
73			game_desc = elm.find('description').text
74			rom = elm.find('rom')
75			rom_name = rom.get('name')
76			rom_crc = rom.get('crc')
77			rom_md5 = rom.get('md5')
78			rom_sha1 = rom.get('sha1')
79			conn.execute("""INSERT INTO tosec_main
80			(game_name, description, rom_name, crc, md5, sha1, system_id)
81			VALUES (?, ?, ?, ?, ?, ?, ?)""",
82			(game_name, game_desc, rom_name, rom_crc, rom_md5, rom_sha1, system_id))
83	nino.borges	806
84			conn.commit()
85			conn.close()
86
87	nino.borges	906
88	nino.borges	805	def AddNewSystem(systemName, shortName, relativePath):
89			"""Adding a single system, with path where the ROMs will live."""
90			conn.execute(f'INSERT INTO main_app_system("name", "short_name", "relative_file_path")VALUES ("{systemName}", "{shortName}", "{relativePath}")')
91			conn.commit()
92			conn.close()
93
94	nino.borges	795	## Next gather game information
95	nino.borges	805	def AddNewRoms(pathToRoms, systemID, testOnly = False):
96	nino.borges	795	"""Adding additional roms."""
97	nino.borges	805	importedCount = 0
98			alreadyExistsCount = 0
99			errImportCount = 0
100
101			sysResult = conn.execute(f'SELECT relative_file_path FROM main_app_system WHERE id={systemID}')
102			for s in sysResult:
103			systemRomPath = s[0]
104			fullRomPath = os.path.join(config.get('RootConfig','softwareBackupStartDir'),systemRomPath)
105
106	nino.borges	795	res = conn.execute('SELECT md5 FROM main_app')
107			currentHashList = []
108			for h in res:
109	nino.borges	805	#print(h[0])
110	nino.borges	795	currentHashList.append(h[0])
111			#print(currentHashList)
112			print(len(currentHashList))
113	nino.borges	805
114	nino.borges	795	hshr = Tool_Box.NinoGenTools.HashFileContents('md5')
115			for testFile in os.listdir(pathToRoms):
116	nino.borges	805	print(f"\n\nNow analyzing {testFile}...")
117	nino.borges	795	hashVal = hshr.HashFile(os.path.join(pathToRoms,testFile))
118			if hashVal in currentHashList:
119			print("This file is already in your collection, skipping.")
120	nino.borges	805	alreadyExistsCount +=1
121	nino.borges	795	else:
122			print("This file is unique. Adding to collection.")
123	nino.borges	806	fileNameInc = 0
124			targetFileName = testFile
125			while os.path.isfile(os.path.join(fullRomPath,targetFileName)):
126			fileNameInc +=1
127			testFileNamePart, testFileExt = os.path.splitext(testFile)
128			targetFileName = testFileNamePart +"_"+ str(fileNameInc) + testFileExt
129			if testOnly:
130			pass
131	nino.borges	805	else:
132	nino.borges	806	shutil.copyfile(os.path.join(pathToRoms,testFile),os.path.join(fullRomPath,targetFileName))
133			conn.execute(f'INSERT INTO main_app("file_name", "system_console", "md5")VALUES("{testFile}","{systemID}","{hashVal}")')
134			conn.commit()
135			importedCount +=1
136	nino.borges	795	conn.close()
137	nino.borges	805	return importedCount, alreadyExistsCount, errImportCount
138	nino.borges	795
139	nino.borges	805	def GetSystemList():
140			res = conn.execute("SELECT id, name, short_name, relative_file_path FROM main_app_system ")
141			systemNamesMatrix = {}
142			for h in res:
143			systemNamesMatrix[h[1]] = [h[0],h[2],h[3]]
144			return systemNamesMatrix
145
146	nino.borges	795	def GetGameListBySystem(systemName):
147	nino.borges	806	res = conn.execute("SELECT main_app.id, main_app.game_name, no_intro_main.game_name, tosec_main.game_name,main_app.file_name FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5 LEFT JOIN tosec_main ON main_app.md5 = tosec_main.md5")
148			#res = conn.execute("SELECT main_app.id, main_app.game_name, no_intro_main.game_name, main_app.file_name FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
149	nino.borges	795	gameNamesMatrix = {}
150			for h in res:
151			if h[1]:
152			gameNamesMatrix[h[1]] = h[0]
153			elif h[2]:
154			gameNamesMatrix[h[2]] = h[0]
155	nino.borges	806	elif h[3]:
156			gameNamesMatrix[h[3]] = h[0]
157	nino.borges	795	else:
158	nino.borges	806	gameNamesMatrix[h[4]] = h[0]
159	nino.borges	795
160			#for g in gameNamesList:
161			# print(g)
162			return gameNamesMatrix
163
164			def GetSingleGameById(id):
165			res = conn.execute(f"""SELECT main_app.game_name, main_app.md5, main_app.file_name, main_app.relative_file_path, no_intro_main.game_name, no_intro_system.name FROM main_app
166			LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5
167			LEFT JOIN no_intro_system ON no_intro_main.no_intro_system_id = no_intro_system.id
168			WHERE main_app.id = {id}""")
169			for h in res:
170			gameTuple = h
171			print(h)
172			return gameTuple
173
174
175
176			if __name__ == '__main__':
177	nino.borges	806	pass
178	nino.borges	805	#test = GetSystemList()
179			#print(test.keys())
180			#AddNewSystem("Super Nintendo", "SNES", r"Roms/Snes/Gromulus")
181	nino.borges	795	#ImportNewNoIntroDat("/home/nino/MyCode/Python/Active_prgs/Gromulus/_data/_dats/Nintendo - Super Nintendo Entertainment System (20230516-033614).dat")
182	nino.borges	806	#AddNewRoms('/mnt/smb/HomeData/ninoborges/Emulation/Roms/Snes/Random2',"1",testOnly=True)
183	nino.borges	795	#res = conn.execute("SELECT * FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
184			#for h in res:
185			# print(h)
186			# print("\n")
187			#GetGameListBySystem('snes')
188	nino.borges	805	#GetSingleGameById('224')
189	nino.borges	795