| 1 |
nino.borges |
976 |
"""Core data access methods for Gromulus."""
|
| 2 |
nino.borges |
795 |
|
| 3 |
nino.borges |
976 |
import configparser
|
| 4 |
|
|
import os
|
| 5 |
|
|
import shutil
|
| 6 |
|
|
import sqlite3
|
| 7 |
nino.borges |
795 |
import xml.etree.ElementTree as ET
|
| 8 |
|
|
|
| 9 |
nino.borges |
976 |
import Tool_Box.NinoGenTools
|
| 10 |
nino.borges |
795 |
|
| 11 |
nino.borges |
976 |
|
| 12 |
nino.borges |
805 |
config = configparser.ConfigParser()
|
| 13 |
|
|
config.read("/home/nino/.gromulus/gromulus_cfg.ini")
|
| 14 |
nino.borges |
795 |
|
| 15 |
nino.borges |
907 |
#conn = sqlite3.connect(config.get('RootConfig','databasePath'))
|
| 16 |
nino.borges |
795 |
|
| 17 |
|
|
|
| 18 |
nino.borges |
907 |
class DatabaseManager:
|
| 19 |
|
|
def __init__(self):
|
| 20 |
|
|
self.db_path = config.get('RootConfig','databasePath') # Hardcoded DB path
|
| 21 |
|
|
self.conn = sqlite3.connect(self.db_path)
|
| 22 |
|
|
|
| 23 |
|
|
def get_connection(self):
|
| 24 |
|
|
return self.conn
|
| 25 |
|
|
|
| 26 |
|
|
def close(self):
|
| 27 |
|
|
if self.conn:
|
| 28 |
|
|
self.conn.close()
|
| 29 |
|
|
self.conn = None
|
| 30 |
|
|
|
| 31 |
|
|
|
| 32 |
|
|
def ImportNewNoIntroDat(datPath, conn):
|
| 33 |
nino.borges |
976 |
"""Replace No-Intro data using a DAT file."""
|
| 34 |
nino.borges |
795 |
tree = ET.parse(datPath)
|
| 35 |
|
|
root = tree.getroot()
|
| 36 |
nino.borges |
976 |
cursor = conn.cursor()
|
| 37 |
nino.borges |
795 |
|
| 38 |
nino.borges |
976 |
header = root.find("header")
|
| 39 |
|
|
if header is None:
|
| 40 |
|
|
raise ValueError("Invalid DAT file: missing <header> section.")
|
| 41 |
nino.borges |
795 |
|
| 42 |
nino.borges |
976 |
headerID = header.findtext("id")
|
| 43 |
|
|
headerName = header.findtext("name")
|
| 44 |
|
|
headerDesc = header.findtext("description")
|
| 45 |
|
|
headerVer = header.findtext("version")
|
| 46 |
nino.borges |
795 |
|
| 47 |
nino.borges |
976 |
if not headerID:
|
| 48 |
|
|
raise ValueError("Invalid DAT file: missing header id.")
|
| 49 |
|
|
|
| 50 |
|
|
cursor.execute("DELETE FROM no_intro_system")
|
| 51 |
|
|
cursor.execute(
|
| 52 |
|
|
"INSERT INTO no_intro_system(id, name, description, dat_version) VALUES (?, ?, ?, ?)",
|
| 53 |
|
|
(int(headerID), headerName, headerDesc, headerVer),
|
| 54 |
|
|
)
|
| 55 |
|
|
|
| 56 |
|
|
cursor.execute("DELETE FROM no_intro_main")
|
| 57 |
|
|
insert_sql = """
|
| 58 |
|
|
INSERT INTO no_intro_main(
|
| 59 |
|
|
game_name, no_intro_id, clone_of_id, description, rom_name,
|
| 60 |
|
|
crc, md5, sha1, sha256, status, no_intro_system_id
|
| 61 |
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
| 62 |
|
|
"""
|
| 63 |
|
|
for elm in root.findall("game"):
|
| 64 |
|
|
gameName = elm.get("name")
|
| 65 |
|
|
gameID = elm.get("id")
|
| 66 |
|
|
gameCloneOfID = elm.get("cloneofid")
|
| 67 |
|
|
gameDesc = elm.findtext("description")
|
| 68 |
|
|
|
| 69 |
|
|
rom = elm.find("rom")
|
| 70 |
|
|
if rom is None:
|
| 71 |
|
|
continue
|
| 72 |
|
|
romName = rom.get("name")
|
| 73 |
|
|
romCRC = rom.get("crc")
|
| 74 |
|
|
romMD5 = rom.get("md5")
|
| 75 |
|
|
romSHA1 = rom.get("sha1")
|
| 76 |
|
|
romSHA256 = rom.get("sha256")
|
| 77 |
|
|
romStatus = rom.get("status")
|
| 78 |
|
|
|
| 79 |
|
|
cursor.execute(
|
| 80 |
|
|
insert_sql,
|
| 81 |
|
|
(
|
| 82 |
|
|
gameName,
|
| 83 |
|
|
gameID,
|
| 84 |
|
|
gameCloneOfID,
|
| 85 |
|
|
gameDesc,
|
| 86 |
|
|
romName,
|
| 87 |
|
|
romCRC,
|
| 88 |
|
|
romMD5,
|
| 89 |
|
|
romSHA1,
|
| 90 |
|
|
romSHA256,
|
| 91 |
|
|
romStatus,
|
| 92 |
|
|
int(headerID),
|
| 93 |
|
|
),
|
| 94 |
|
|
)
|
| 95 |
|
|
|
| 96 |
nino.borges |
795 |
conn.commit()
|
| 97 |
|
|
|
| 98 |
|
|
|
| 99 |
nino.borges |
907 |
def ImportNewTosecDat(datPath, conn):
|
| 100 |
nino.borges |
976 |
"""Replace TOSEC data using a DAT file."""
|
| 101 |
nino.borges |
907 |
cursor = conn.cursor()
|
| 102 |
nino.borges |
806 |
tree = ET.parse(datPath)
|
| 103 |
|
|
root = tree.getroot()
|
| 104 |
|
|
|
| 105 |
nino.borges |
907 |
header = root.find("header")
|
| 106 |
nino.borges |
976 |
if header is None:
|
| 107 |
|
|
raise ValueError("Invalid DAT file: missing <header> section.")
|
| 108 |
|
|
system_name = header.findtext("name") or ""
|
| 109 |
nino.borges |
906 |
|
| 110 |
nino.borges |
976 |
# TOSEC table currently references no_intro_system ids.
|
| 111 |
|
|
system_row = cursor.execute(
|
| 112 |
|
|
"SELECT id FROM no_intro_system WHERE lower(name)=lower(?)",
|
| 113 |
|
|
(system_name,),
|
| 114 |
|
|
).fetchone()
|
| 115 |
|
|
if system_row:
|
| 116 |
|
|
system_id = system_row[0]
|
| 117 |
|
|
else:
|
| 118 |
|
|
max_id_row = cursor.execute("SELECT COALESCE(MAX(id), 0) FROM no_intro_system").fetchone()
|
| 119 |
|
|
system_id = (max_id_row[0] or 0) + 1
|
| 120 |
|
|
cursor.execute(
|
| 121 |
|
|
"INSERT INTO no_intro_system(id, name, description, dat_version) VALUES (?, ?, ?, ?)",
|
| 122 |
|
|
(system_id, system_name, "Imported from TOSEC header", None),
|
| 123 |
|
|
)
|
| 124 |
nino.borges |
906 |
|
| 125 |
nino.borges |
907 |
cursor.execute("DELETE FROM tosec_main")
|
| 126 |
nino.borges |
976 |
insert_sql = """
|
| 127 |
|
|
INSERT INTO tosec_main(game_name, description, rom_name, crc, md5, sha1, system_id)
|
| 128 |
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
| 129 |
|
|
"""
|
| 130 |
nino.borges |
907 |
for game in root.findall("game"):
|
| 131 |
|
|
game_name = game.attrib.get("name", "")
|
| 132 |
nino.borges |
976 |
game_desc = game.findtext("description")
|
| 133 |
nino.borges |
907 |
rom = game.find("rom")
|
| 134 |
|
|
if rom is None:
|
| 135 |
|
|
continue
|
| 136 |
nino.borges |
976 |
cursor.execute(
|
| 137 |
|
|
insert_sql,
|
| 138 |
|
|
(
|
| 139 |
|
|
game_name,
|
| 140 |
|
|
game_desc,
|
| 141 |
|
|
rom.attrib.get("name", ""),
|
| 142 |
|
|
rom.attrib.get("crc", ""),
|
| 143 |
|
|
rom.attrib.get("md5", ""),
|
| 144 |
|
|
rom.attrib.get("sha1", ""),
|
| 145 |
|
|
system_id,
|
| 146 |
|
|
),
|
| 147 |
nino.borges |
907 |
)
|
| 148 |
|
|
|
| 149 |
nino.borges |
806 |
conn.commit()
|
| 150 |
nino.borges |
907 |
print("TOSEC DAT imported successfully.")
|
| 151 |
nino.borges |
806 |
|
| 152 |
nino.borges |
906 |
|
| 153 |
nino.borges |
907 |
def AddNewSystem(systemName, shortName, relativePath, conn):
|
| 154 |
nino.borges |
805 |
"""Adding a single system, with path where the ROMs will live."""
|
| 155 |
nino.borges |
976 |
conn.execute(
|
| 156 |
|
|
'INSERT INTO main_app_system("name", "short_name", "relative_file_path") VALUES (?, ?, ?)',
|
| 157 |
|
|
(systemName, shortName, relativePath),
|
| 158 |
|
|
)
|
| 159 |
nino.borges |
805 |
conn.commit()
|
| 160 |
|
|
|
| 161 |
nino.borges |
976 |
|
| 162 |
nino.borges |
907 |
def AddNewRoms(pathToRoms, systemID, conn, testOnly = False):
|
| 163 |
nino.borges |
795 |
"""Adding additional roms."""
|
| 164 |
nino.borges |
805 |
importedCount = 0
|
| 165 |
|
|
alreadyExistsCount = 0
|
| 166 |
|
|
errImportCount = 0
|
| 167 |
|
|
|
| 168 |
nino.borges |
976 |
system_row = conn.execute(
|
| 169 |
|
|
"SELECT relative_file_path FROM main_app_system WHERE id = ?",
|
| 170 |
|
|
(systemID,),
|
| 171 |
|
|
).fetchone()
|
| 172 |
|
|
if not system_row:
|
| 173 |
|
|
raise ValueError(f"System id {systemID} not found in main_app_system.")
|
| 174 |
|
|
|
| 175 |
|
|
systemRomPath = system_row[0]
|
| 176 |
|
|
fullRomPath = os.path.join(config.get("RootConfig", "softwareBackupStartDir"), systemRomPath)
|
| 177 |
|
|
os.makedirs(fullRomPath, exist_ok=True)
|
| 178 |
|
|
|
| 179 |
|
|
currentHashList = {
|
| 180 |
|
|
row[0]
|
| 181 |
|
|
for row in conn.execute("SELECT container_md5_hash FROM main_app WHERE container_md5_hash IS NOT NULL")
|
| 182 |
|
|
}
|
| 183 |
nino.borges |
795 |
print(len(currentHashList))
|
| 184 |
nino.borges |
805 |
|
| 185 |
nino.borges |
976 |
hshr = Tool_Box.NinoGenTools.HashFileContents("md5")
|
| 186 |
nino.borges |
795 |
for testFile in os.listdir(pathToRoms):
|
| 187 |
nino.borges |
976 |
source_path = os.path.join(pathToRoms, testFile)
|
| 188 |
|
|
if not os.path.isfile(source_path):
|
| 189 |
|
|
continue
|
| 190 |
|
|
|
| 191 |
nino.borges |
805 |
print(f"\n\nNow analyzing {testFile}...")
|
| 192 |
nino.borges |
976 |
hashVal = hshr.HashFile(source_path)
|
| 193 |
nino.borges |
795 |
if hashVal in currentHashList:
|
| 194 |
|
|
print("This file is already in your collection, skipping.")
|
| 195 |
nino.borges |
805 |
alreadyExistsCount +=1
|
| 196 |
nino.borges |
795 |
else:
|
| 197 |
|
|
print("This file is unique. Adding to collection.")
|
| 198 |
nino.borges |
806 |
fileNameInc = 0
|
| 199 |
|
|
targetFileName = testFile
|
| 200 |
|
|
while os.path.isfile(os.path.join(fullRomPath,targetFileName)):
|
| 201 |
|
|
fileNameInc +=1
|
| 202 |
|
|
testFileNamePart, testFileExt = os.path.splitext(testFile)
|
| 203 |
|
|
targetFileName = testFileNamePart +"_"+ str(fileNameInc) + testFileExt
|
| 204 |
|
|
if testOnly:
|
| 205 |
|
|
pass
|
| 206 |
nino.borges |
805 |
else:
|
| 207 |
nino.borges |
976 |
try:
|
| 208 |
|
|
shutil.copyfile(source_path, os.path.join(fullRomPath, targetFileName))
|
| 209 |
|
|
conn.execute(
|
| 210 |
|
|
'INSERT INTO main_app("container_file_name", "system_console", "container_md5_hash") VALUES(?, ?, ?)',
|
| 211 |
|
|
(targetFileName, systemID, hashVal),
|
| 212 |
|
|
)
|
| 213 |
|
|
conn.commit()
|
| 214 |
|
|
except Exception:
|
| 215 |
|
|
errImportCount += 1
|
| 216 |
|
|
continue
|
| 217 |
|
|
currentHashList.add(hashVal)
|
| 218 |
nino.borges |
806 |
importedCount +=1
|
| 219 |
nino.borges |
805 |
return importedCount, alreadyExistsCount, errImportCount
|
| 220 |
nino.borges |
795 |
|
| 221 |
nino.borges |
976 |
|
| 222 |
nino.borges |
907 |
def GetSystemList(conn):
|
| 223 |
nino.borges |
805 |
res = conn.execute("SELECT id, name, short_name, relative_file_path FROM main_app_system ")
|
| 224 |
|
|
systemNamesMatrix = {}
|
| 225 |
|
|
for h in res:
|
| 226 |
|
|
systemNamesMatrix[h[1]] = [h[0],h[2],h[3]]
|
| 227 |
|
|
return systemNamesMatrix
|
| 228 |
|
|
|
| 229 |
nino.borges |
907 |
def GetGameListBySystem(systemName, conn):
|
| 230 |
nino.borges |
976 |
system_row = conn.execute(
|
| 231 |
|
|
"""
|
| 232 |
|
|
SELECT id
|
| 233 |
|
|
FROM main_app_system
|
| 234 |
|
|
WHERE lower(short_name) = lower(?)
|
| 235 |
|
|
OR lower(name) = lower(?)
|
| 236 |
|
|
""",
|
| 237 |
|
|
(systemName, systemName),
|
| 238 |
|
|
).fetchone()
|
| 239 |
|
|
if not system_row:
|
| 240 |
|
|
return {}
|
| 241 |
|
|
|
| 242 |
|
|
system_id = system_row[0]
|
| 243 |
|
|
res = conn.execute(
|
| 244 |
|
|
"""
|
| 245 |
|
|
SELECT ma.id,
|
| 246 |
|
|
ma.game_name,
|
| 247 |
|
|
nim.game_name,
|
| 248 |
|
|
tm.game_name,
|
| 249 |
|
|
COALESCE(fh.file_name, ma.container_file_name)
|
| 250 |
|
|
FROM main_app ma
|
| 251 |
|
|
LEFT JOIN main_app_file_hash fh
|
| 252 |
|
|
ON fh.id = (
|
| 253 |
|
|
SELECT MIN(fh2.id)
|
| 254 |
|
|
FROM main_app_file_hash fh2
|
| 255 |
|
|
WHERE fh2.container_file_id = ma.id
|
| 256 |
|
|
)
|
| 257 |
|
|
LEFT JOIN no_intro_main nim ON lower(fh.file_md5_hash) = lower(nim.md5)
|
| 258 |
|
|
LEFT JOIN tosec_main tm ON lower(fh.file_md5_hash) = lower(tm.md5)
|
| 259 |
|
|
WHERE ma.system_console = ?
|
| 260 |
|
|
""",
|
| 261 |
|
|
(system_id,),
|
| 262 |
|
|
)
|
| 263 |
nino.borges |
795 |
gameNamesMatrix = {}
|
| 264 |
nino.borges |
976 |
duplicate_counter = {}
|
| 265 |
nino.borges |
795 |
for h in res:
|
| 266 |
|
|
if h[1]:
|
| 267 |
nino.borges |
976 |
display_name = h[1]
|
| 268 |
nino.borges |
795 |
elif h[2]:
|
| 269 |
nino.borges |
976 |
display_name = h[2]
|
| 270 |
nino.borges |
806 |
elif h[3]:
|
| 271 |
nino.borges |
976 |
display_name = h[3]
|
| 272 |
nino.borges |
795 |
else:
|
| 273 |
nino.borges |
976 |
display_name = h[4] or f"Unknown-{h[0]}"
|
| 274 |
nino.borges |
795 |
|
| 275 |
nino.borges |
976 |
# Keep titles unique in the listbox mapping.
|
| 276 |
|
|
if display_name in gameNamesMatrix:
|
| 277 |
|
|
duplicate_counter[display_name] = duplicate_counter.get(display_name, 1) + 1
|
| 278 |
|
|
unique_name = f"{display_name} ({duplicate_counter[display_name]})"
|
| 279 |
|
|
gameNamesMatrix[unique_name] = h[0]
|
| 280 |
|
|
else:
|
| 281 |
|
|
duplicate_counter[display_name] = 1
|
| 282 |
|
|
gameNamesMatrix[display_name] = h[0]
|
| 283 |
|
|
|
| 284 |
nino.borges |
795 |
return gameNamesMatrix
|
| 285 |
|
|
|
| 286 |
nino.borges |
976 |
|
| 287 |
nino.borges |
907 |
def GetSingleGameById(game_id, conn):
|
| 288 |
|
|
"""Fetches full metadata for a single ROM entry, including No-Intro and TOSEC info if available."""
|
| 289 |
|
|
cursor = conn.cursor()
|
| 290 |
|
|
|
| 291 |
|
|
cursor.execute("""
|
| 292 |
nino.borges |
976 |
SELECT ma.id,
|
| 293 |
|
|
ma.game_name,
|
| 294 |
|
|
ma.container_file_name,
|
| 295 |
|
|
sys.name,
|
| 296 |
|
|
sys.short_name,
|
| 297 |
|
|
sys.relative_file_path,
|
| 298 |
|
|
ma.container_md5_hash
|
| 299 |
nino.borges |
907 |
FROM main_app ma
|
| 300 |
nino.borges |
976 |
LEFT JOIN main_app_system sys ON ma.system_console = sys.id
|
| 301 |
nino.borges |
907 |
WHERE ma.id = ?
|
| 302 |
|
|
""", (game_id,))
|
| 303 |
|
|
game = cursor.fetchone()
|
| 304 |
nino.borges |
976 |
if not game:
|
| 305 |
|
|
return None
|
| 306 |
nino.borges |
907 |
|
| 307 |
nino.borges |
976 |
file_hash_row = cursor.execute(
|
| 308 |
|
|
"""
|
| 309 |
|
|
SELECT file_name, file_md5_hash
|
| 310 |
|
|
FROM main_app_file_hash
|
| 311 |
|
|
WHERE container_file_id = ?
|
| 312 |
|
|
ORDER BY id ASC
|
| 313 |
|
|
LIMIT 1
|
| 314 |
|
|
""",
|
| 315 |
|
|
(game[0],),
|
| 316 |
|
|
).fetchone()
|
| 317 |
|
|
|
| 318 |
|
|
resolved_file_name = file_hash_row[0] if file_hash_row and file_hash_row[0] else game[2]
|
| 319 |
|
|
resolved_hash = file_hash_row[1] if file_hash_row and file_hash_row[1] else game[6]
|
| 320 |
|
|
|
| 321 |
nino.borges |
907 |
no_intro = cursor.execute("""
|
| 322 |
nino.borges |
976 |
SELECT nis.name, nim.game_name
|
| 323 |
nino.borges |
907 |
FROM no_intro_main nim
|
| 324 |
nino.borges |
976 |
JOIN no_intro_system nis ON nim.no_intro_system_id = nis.id
|
| 325 |
|
|
WHERE lower(nim.md5) = lower(?)
|
| 326 |
|
|
""", (resolved_hash,)).fetchone()
|
| 327 |
nino.borges |
907 |
|
| 328 |
|
|
tosec = cursor.execute("""
|
| 329 |
nino.borges |
976 |
SELECT nis.name, tm.game_name
|
| 330 |
nino.borges |
907 |
FROM tosec_main tm
|
| 331 |
nino.borges |
976 |
LEFT JOIN no_intro_system nis ON tm.system_id = nis.id
|
| 332 |
|
|
WHERE lower(tm.md5) = lower(?)
|
| 333 |
|
|
""", (resolved_hash,)).fetchone()
|
| 334 |
nino.borges |
907 |
|
| 335 |
nino.borges |
976 |
if game[5]:
|
| 336 |
|
|
file_path = os.path.join(
|
| 337 |
|
|
config.get("RootConfig", "softwareBackupStartDir"),
|
| 338 |
|
|
game[5],
|
| 339 |
|
|
resolved_file_name or "",
|
| 340 |
|
|
)
|
| 341 |
|
|
else:
|
| 342 |
|
|
file_path = resolved_file_name or ""
|
| 343 |
nino.borges |
907 |
|
| 344 |
|
|
return {
|
| 345 |
|
|
"id": game[0],
|
| 346 |
nino.borges |
976 |
"game_name": game[1],
|
| 347 |
|
|
"hash": resolved_hash,
|
| 348 |
|
|
"filename": resolved_file_name,
|
| 349 |
|
|
"path": file_path,
|
| 350 |
|
|
"system_name": game[3],
|
| 351 |
|
|
"system_short_name": game[4],
|
| 352 |
nino.borges |
907 |
"no_intro_system": no_intro[0] if no_intro else None,
|
| 353 |
|
|
"no_intro_game": no_intro[1] if no_intro else None,
|
| 354 |
|
|
"tosec_system": tosec[0] if tosec else None,
|
| 355 |
|
|
"tosec_game": tosec[1] if tosec else None,
|
| 356 |
|
|
}
|
| 357 |
|
|
|
| 358 |
nino.borges |
795 |
|
| 359 |
|
|
|
| 360 |
|
|
|
| 361 |
|
|
if __name__ == '__main__':
|
| 362 |
nino.borges |
806 |
pass
|
| 363 |
nino.borges |
805 |
#test = GetSystemList()
|
| 364 |
|
|
#print(test.keys())
|
| 365 |
|
|
#AddNewSystem("Super Nintendo", "SNES", r"Roms/Snes/Gromulus")
|
| 366 |
nino.borges |
795 |
#ImportNewNoIntroDat("/home/nino/MyCode/Python/Active_prgs/Gromulus/_data/_dats/Nintendo - Super Nintendo Entertainment System (20230516-033614).dat")
|
| 367 |
nino.borges |
806 |
#AddNewRoms('/mnt/smb/HomeData/ninoborges/Emulation/Roms/Snes/Random2',"1",testOnly=True)
|
| 368 |
nino.borges |
795 |
#res = conn.execute("SELECT * FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
|
| 369 |
|
|
#for h in res:
|
| 370 |
|
|
# print(h)
|
| 371 |
|
|
# print("\n")
|
| 372 |
|
|
#GetGameListBySystem('snes')
|
| 373 |
nino.borges |
805 |
#GetSingleGameById('224')
|