ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/ns_dev/Python/NinoCode/Active_prgs/Gromulus/Gromulus_Lib.py
Revision: 976
Committed: Fri Feb 13 02:00:16 2026 UTC (6 weeks, 1 day ago) by nino.borges
Content type: text/x-python
File size: 11999 byte(s)
Log Message:
Refactor catalog data flow: unify schema usage, fix UI data contracts, and match metadata via main_app_file_hash

File Contents

# User Rev Content
1 nino.borges 976 """Core data access methods for Gromulus."""
2 nino.borges 795
3 nino.borges 976 import configparser
4     import os
5     import shutil
6     import sqlite3
7 nino.borges 795 import xml.etree.ElementTree as ET
8    
9 nino.borges 976 import Tool_Box.NinoGenTools
10 nino.borges 795
11 nino.borges 976
12 nino.borges 805 config = configparser.ConfigParser()
13     config.read("/home/nino/.gromulus/gromulus_cfg.ini")
14 nino.borges 795
15 nino.borges 907 #conn = sqlite3.connect(config.get('RootConfig','databasePath'))
16 nino.borges 795
17    
18 nino.borges 907 class DatabaseManager:
19     def __init__(self):
20     self.db_path = config.get('RootConfig','databasePath') # Hardcoded DB path
21     self.conn = sqlite3.connect(self.db_path)
22    
23     def get_connection(self):
24     return self.conn
25    
26     def close(self):
27     if self.conn:
28     self.conn.close()
29     self.conn = None
30    
31    
32     def ImportNewNoIntroDat(datPath, conn):
33 nino.borges 976 """Replace No-Intro data using a DAT file."""
34 nino.borges 795 tree = ET.parse(datPath)
35     root = tree.getroot()
36 nino.borges 976 cursor = conn.cursor()
37 nino.borges 795
38 nino.borges 976 header = root.find("header")
39     if header is None:
40     raise ValueError("Invalid DAT file: missing <header> section.")
41 nino.borges 795
42 nino.borges 976 headerID = header.findtext("id")
43     headerName = header.findtext("name")
44     headerDesc = header.findtext("description")
45     headerVer = header.findtext("version")
46 nino.borges 795
47 nino.borges 976 if not headerID:
48     raise ValueError("Invalid DAT file: missing header id.")
49    
50     cursor.execute("DELETE FROM no_intro_system")
51     cursor.execute(
52     "INSERT INTO no_intro_system(id, name, description, dat_version) VALUES (?, ?, ?, ?)",
53     (int(headerID), headerName, headerDesc, headerVer),
54     )
55    
56     cursor.execute("DELETE FROM no_intro_main")
57     insert_sql = """
58     INSERT INTO no_intro_main(
59     game_name, no_intro_id, clone_of_id, description, rom_name,
60     crc, md5, sha1, sha256, status, no_intro_system_id
61     ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
62     """
63     for elm in root.findall("game"):
64     gameName = elm.get("name")
65     gameID = elm.get("id")
66     gameCloneOfID = elm.get("cloneofid")
67     gameDesc = elm.findtext("description")
68    
69     rom = elm.find("rom")
70     if rom is None:
71     continue
72     romName = rom.get("name")
73     romCRC = rom.get("crc")
74     romMD5 = rom.get("md5")
75     romSHA1 = rom.get("sha1")
76     romSHA256 = rom.get("sha256")
77     romStatus = rom.get("status")
78    
79     cursor.execute(
80     insert_sql,
81     (
82     gameName,
83     gameID,
84     gameCloneOfID,
85     gameDesc,
86     romName,
87     romCRC,
88     romMD5,
89     romSHA1,
90     romSHA256,
91     romStatus,
92     int(headerID),
93     ),
94     )
95    
96 nino.borges 795 conn.commit()
97    
98    
99 nino.borges 907 def ImportNewTosecDat(datPath, conn):
100 nino.borges 976 """Replace TOSEC data using a DAT file."""
101 nino.borges 907 cursor = conn.cursor()
102 nino.borges 806 tree = ET.parse(datPath)
103     root = tree.getroot()
104    
105 nino.borges 907 header = root.find("header")
106 nino.borges 976 if header is None:
107     raise ValueError("Invalid DAT file: missing <header> section.")
108     system_name = header.findtext("name") or ""
109 nino.borges 906
110 nino.borges 976 # TOSEC table currently references no_intro_system ids.
111     system_row = cursor.execute(
112     "SELECT id FROM no_intro_system WHERE lower(name)=lower(?)",
113     (system_name,),
114     ).fetchone()
115     if system_row:
116     system_id = system_row[0]
117     else:
118     max_id_row = cursor.execute("SELECT COALESCE(MAX(id), 0) FROM no_intro_system").fetchone()
119     system_id = (max_id_row[0] or 0) + 1
120     cursor.execute(
121     "INSERT INTO no_intro_system(id, name, description, dat_version) VALUES (?, ?, ?, ?)",
122     (system_id, system_name, "Imported from TOSEC header", None),
123     )
124 nino.borges 906
125 nino.borges 907 cursor.execute("DELETE FROM tosec_main")
126 nino.borges 976 insert_sql = """
127     INSERT INTO tosec_main(game_name, description, rom_name, crc, md5, sha1, system_id)
128     VALUES (?, ?, ?, ?, ?, ?, ?)
129     """
130 nino.borges 907 for game in root.findall("game"):
131     game_name = game.attrib.get("name", "")
132 nino.borges 976 game_desc = game.findtext("description")
133 nino.borges 907 rom = game.find("rom")
134     if rom is None:
135     continue
136 nino.borges 976 cursor.execute(
137     insert_sql,
138     (
139     game_name,
140     game_desc,
141     rom.attrib.get("name", ""),
142     rom.attrib.get("crc", ""),
143     rom.attrib.get("md5", ""),
144     rom.attrib.get("sha1", ""),
145     system_id,
146     ),
147 nino.borges 907 )
148    
149 nino.borges 806 conn.commit()
150 nino.borges 907 print("TOSEC DAT imported successfully.")
151 nino.borges 806
152 nino.borges 906
153 nino.borges 907 def AddNewSystem(systemName, shortName, relativePath, conn):
154 nino.borges 805 """Adding a single system, with path where the ROMs will live."""
155 nino.borges 976 conn.execute(
156     'INSERT INTO main_app_system("name", "short_name", "relative_file_path") VALUES (?, ?, ?)',
157     (systemName, shortName, relativePath),
158     )
159 nino.borges 805 conn.commit()
160    
161 nino.borges 976
162 nino.borges 907 def AddNewRoms(pathToRoms, systemID, conn, testOnly = False):
163 nino.borges 795 """Adding additional roms."""
164 nino.borges 805 importedCount = 0
165     alreadyExistsCount = 0
166     errImportCount = 0
167    
168 nino.borges 976 system_row = conn.execute(
169     "SELECT relative_file_path FROM main_app_system WHERE id = ?",
170     (systemID,),
171     ).fetchone()
172     if not system_row:
173     raise ValueError(f"System id {systemID} not found in main_app_system.")
174    
175     systemRomPath = system_row[0]
176     fullRomPath = os.path.join(config.get("RootConfig", "softwareBackupStartDir"), systemRomPath)
177     os.makedirs(fullRomPath, exist_ok=True)
178    
179     currentHashList = {
180     row[0]
181     for row in conn.execute("SELECT container_md5_hash FROM main_app WHERE container_md5_hash IS NOT NULL")
182     }
183 nino.borges 795 print(len(currentHashList))
184 nino.borges 805
185 nino.borges 976 hshr = Tool_Box.NinoGenTools.HashFileContents("md5")
186 nino.borges 795 for testFile in os.listdir(pathToRoms):
187 nino.borges 976 source_path = os.path.join(pathToRoms, testFile)
188     if not os.path.isfile(source_path):
189     continue
190    
191 nino.borges 805 print(f"\n\nNow analyzing {testFile}...")
192 nino.borges 976 hashVal = hshr.HashFile(source_path)
193 nino.borges 795 if hashVal in currentHashList:
194     print("This file is already in your collection, skipping.")
195 nino.borges 805 alreadyExistsCount +=1
196 nino.borges 795 else:
197     print("This file is unique. Adding to collection.")
198 nino.borges 806 fileNameInc = 0
199     targetFileName = testFile
200     while os.path.isfile(os.path.join(fullRomPath,targetFileName)):
201     fileNameInc +=1
202     testFileNamePart, testFileExt = os.path.splitext(testFile)
203     targetFileName = testFileNamePart +"_"+ str(fileNameInc) + testFileExt
204     if testOnly:
205     pass
206 nino.borges 805 else:
207 nino.borges 976 try:
208     shutil.copyfile(source_path, os.path.join(fullRomPath, targetFileName))
209     conn.execute(
210     'INSERT INTO main_app("container_file_name", "system_console", "container_md5_hash") VALUES(?, ?, ?)',
211     (targetFileName, systemID, hashVal),
212     )
213     conn.commit()
214     except Exception:
215     errImportCount += 1
216     continue
217     currentHashList.add(hashVal)
218 nino.borges 806 importedCount +=1
219 nino.borges 805 return importedCount, alreadyExistsCount, errImportCount
220 nino.borges 795
221 nino.borges 976
222 nino.borges 907 def GetSystemList(conn):
223 nino.borges 805 res = conn.execute("SELECT id, name, short_name, relative_file_path FROM main_app_system ")
224     systemNamesMatrix = {}
225     for h in res:
226     systemNamesMatrix[h[1]] = [h[0],h[2],h[3]]
227     return systemNamesMatrix
228    
229 nino.borges 907 def GetGameListBySystem(systemName, conn):
230 nino.borges 976 system_row = conn.execute(
231     """
232     SELECT id
233     FROM main_app_system
234     WHERE lower(short_name) = lower(?)
235     OR lower(name) = lower(?)
236     """,
237     (systemName, systemName),
238     ).fetchone()
239     if not system_row:
240     return {}
241    
242     system_id = system_row[0]
243     res = conn.execute(
244     """
245     SELECT ma.id,
246     ma.game_name,
247     nim.game_name,
248     tm.game_name,
249     COALESCE(fh.file_name, ma.container_file_name)
250     FROM main_app ma
251     LEFT JOIN main_app_file_hash fh
252     ON fh.id = (
253     SELECT MIN(fh2.id)
254     FROM main_app_file_hash fh2
255     WHERE fh2.container_file_id = ma.id
256     )
257     LEFT JOIN no_intro_main nim ON lower(fh.file_md5_hash) = lower(nim.md5)
258     LEFT JOIN tosec_main tm ON lower(fh.file_md5_hash) = lower(tm.md5)
259     WHERE ma.system_console = ?
260     """,
261     (system_id,),
262     )
263 nino.borges 795 gameNamesMatrix = {}
264 nino.borges 976 duplicate_counter = {}
265 nino.borges 795 for h in res:
266     if h[1]:
267 nino.borges 976 display_name = h[1]
268 nino.borges 795 elif h[2]:
269 nino.borges 976 display_name = h[2]
270 nino.borges 806 elif h[3]:
271 nino.borges 976 display_name = h[3]
272 nino.borges 795 else:
273 nino.borges 976 display_name = h[4] or f"Unknown-{h[0]}"
274 nino.borges 795
275 nino.borges 976 # Keep titles unique in the listbox mapping.
276     if display_name in gameNamesMatrix:
277     duplicate_counter[display_name] = duplicate_counter.get(display_name, 1) + 1
278     unique_name = f"{display_name} ({duplicate_counter[display_name]})"
279     gameNamesMatrix[unique_name] = h[0]
280     else:
281     duplicate_counter[display_name] = 1
282     gameNamesMatrix[display_name] = h[0]
283    
284 nino.borges 795 return gameNamesMatrix
285    
286 nino.borges 976
287 nino.borges 907 def GetSingleGameById(game_id, conn):
288     """Fetches full metadata for a single ROM entry, including No-Intro and TOSEC info if available."""
289     cursor = conn.cursor()
290    
291     cursor.execute("""
292 nino.borges 976 SELECT ma.id,
293     ma.game_name,
294     ma.container_file_name,
295     sys.name,
296     sys.short_name,
297     sys.relative_file_path,
298     ma.container_md5_hash
299 nino.borges 907 FROM main_app ma
300 nino.borges 976 LEFT JOIN main_app_system sys ON ma.system_console = sys.id
301 nino.borges 907 WHERE ma.id = ?
302     """, (game_id,))
303     game = cursor.fetchone()
304 nino.borges 976 if not game:
305     return None
306 nino.borges 907
307 nino.borges 976 file_hash_row = cursor.execute(
308     """
309     SELECT file_name, file_md5_hash
310     FROM main_app_file_hash
311     WHERE container_file_id = ?
312     ORDER BY id ASC
313     LIMIT 1
314     """,
315     (game[0],),
316     ).fetchone()
317    
318     resolved_file_name = file_hash_row[0] if file_hash_row and file_hash_row[0] else game[2]
319     resolved_hash = file_hash_row[1] if file_hash_row and file_hash_row[1] else game[6]
320    
321 nino.borges 907 no_intro = cursor.execute("""
322 nino.borges 976 SELECT nis.name, nim.game_name
323 nino.borges 907 FROM no_intro_main nim
324 nino.borges 976 JOIN no_intro_system nis ON nim.no_intro_system_id = nis.id
325     WHERE lower(nim.md5) = lower(?)
326     """, (resolved_hash,)).fetchone()
327 nino.borges 907
328     tosec = cursor.execute("""
329 nino.borges 976 SELECT nis.name, tm.game_name
330 nino.borges 907 FROM tosec_main tm
331 nino.borges 976 LEFT JOIN no_intro_system nis ON tm.system_id = nis.id
332     WHERE lower(tm.md5) = lower(?)
333     """, (resolved_hash,)).fetchone()
334 nino.borges 907
335 nino.borges 976 if game[5]:
336     file_path = os.path.join(
337     config.get("RootConfig", "softwareBackupStartDir"),
338     game[5],
339     resolved_file_name or "",
340     )
341     else:
342     file_path = resolved_file_name or ""
343 nino.borges 907
344     return {
345     "id": game[0],
346 nino.borges 976 "game_name": game[1],
347     "hash": resolved_hash,
348     "filename": resolved_file_name,
349     "path": file_path,
350     "system_name": game[3],
351     "system_short_name": game[4],
352 nino.borges 907 "no_intro_system": no_intro[0] if no_intro else None,
353     "no_intro_game": no_intro[1] if no_intro else None,
354     "tosec_system": tosec[0] if tosec else None,
355     "tosec_game": tosec[1] if tosec else None,
356     }
357    
358 nino.borges 795
359    
360    
361     if __name__ == '__main__':
362 nino.borges 806 pass
363 nino.borges 805 #test = GetSystemList()
364     #print(test.keys())
365     #AddNewSystem("Super Nintendo", "SNES", r"Roms/Snes/Gromulus")
366 nino.borges 795 #ImportNewNoIntroDat("/home/nino/MyCode/Python/Active_prgs/Gromulus/_data/_dats/Nintendo - Super Nintendo Entertainment System (20230516-033614).dat")
367 nino.borges 806 #AddNewRoms('/mnt/smb/HomeData/ninoborges/Emulation/Roms/Snes/Random2',"1",testOnly=True)
368 nino.borges 795 #res = conn.execute("SELECT * FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
369     #for h in res:
370     # print(h)
371     # print("\n")
372     #GetGameListBySystem('snes')
373 nino.borges 805 #GetSingleGameById('224')