| 1 |
nino.borges |
976 |
"""Core data access methods for Gromulus."""
|
| 2 |
nino.borges |
795 |
|
| 3 |
nino.borges |
976 |
import configparser
|
| 4 |
|
|
import os
|
| 5 |
|
|
import shutil
|
| 6 |
|
|
import sqlite3
|
| 7 |
nino.borges |
795 |
import xml.etree.ElementTree as ET
|
| 8 |
|
|
|
| 9 |
nino.borges |
976 |
import Tool_Box.NinoGenTools
|
| 10 |
nino.borges |
795 |
|
| 11 |
nino.borges |
976 |
|
| 12 |
nino.borges |
805 |
config = configparser.ConfigParser()
|
| 13 |
|
|
config.read("/home/nino/.gromulus/gromulus_cfg.ini")
|
| 14 |
nino.borges |
795 |
|
| 15 |
nino.borges |
907 |
#conn = sqlite3.connect(config.get('RootConfig','databasePath'))
|
| 16 |
nino.borges |
795 |
|
| 17 |
|
|
|
| 18 |
nino.borges |
907 |
class DatabaseManager:
|
| 19 |
|
|
def __init__(self):
|
| 20 |
|
|
self.db_path = config.get('RootConfig','databasePath') # Hardcoded DB path
|
| 21 |
|
|
self.conn = sqlite3.connect(self.db_path)
|
| 22 |
|
|
|
| 23 |
|
|
def get_connection(self):
|
| 24 |
|
|
return self.conn
|
| 25 |
|
|
|
| 26 |
|
|
def close(self):
|
| 27 |
|
|
if self.conn:
|
| 28 |
|
|
self.conn.close()
|
| 29 |
|
|
self.conn = None
|
| 30 |
|
|
|
| 31 |
|
|
|
| 32 |
nino.borges |
978 |
def _column_exists(conn, table_name, column_name):
|
| 33 |
|
|
columns = [row[1] for row in conn.execute(f"PRAGMA table_info({table_name})")]
|
| 34 |
|
|
return column_name in columns
|
| 35 |
|
|
|
| 36 |
|
|
|
| 37 |
|
|
def EnsureMetadataSchema(conn):
|
| 38 |
|
|
"""Adds metadata-scoped columns/tables required by current import and matching workflow."""
|
| 39 |
|
|
conn.execute(
|
| 40 |
|
|
"""
|
| 41 |
|
|
CREATE TABLE IF NOT EXISTS dat_import_history(
|
| 42 |
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 43 |
|
|
app_system_id INTEGER NOT NULL,
|
| 44 |
|
|
source TEXT NOT NULL,
|
| 45 |
|
|
dat_name TEXT,
|
| 46 |
|
|
dat_description TEXT,
|
| 47 |
|
|
dat_version TEXT,
|
| 48 |
|
|
imported_at TEXT NOT NULL,
|
| 49 |
|
|
entry_count INTEGER NOT NULL DEFAULT 0,
|
| 50 |
|
|
UNIQUE(app_system_id, source),
|
| 51 |
|
|
FOREIGN KEY(app_system_id) REFERENCES main_app_system(id)
|
| 52 |
|
|
)
|
| 53 |
|
|
"""
|
| 54 |
|
|
)
|
| 55 |
|
|
|
| 56 |
|
|
if not _column_exists(conn, "no_intro_main", "app_system_id"):
|
| 57 |
|
|
conn.execute("ALTER TABLE no_intro_main ADD COLUMN app_system_id INTEGER")
|
| 58 |
|
|
if not _column_exists(conn, "tosec_main", "app_system_id"):
|
| 59 |
|
|
conn.execute("ALTER TABLE tosec_main ADD COLUMN app_system_id INTEGER")
|
| 60 |
nino.borges |
990 |
if not _column_exists(conn, "main_app", "favorite_game"):
|
| 61 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN favorite_game INTEGER")
|
| 62 |
|
|
if not _column_exists(conn, "main_app", "release_date"):
|
| 63 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN release_date TEXT")
|
| 64 |
|
|
if not _column_exists(conn, "main_app", "release_date_scraped"):
|
| 65 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN release_date_scraped TEXT")
|
| 66 |
|
|
if not _column_exists(conn, "main_app", "game_genre"):
|
| 67 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN game_genre TEXT")
|
| 68 |
|
|
if not _column_exists(conn, "main_app", "game_genre_scraped"):
|
| 69 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN game_genre_scraped TEXT")
|
| 70 |
|
|
if not _column_exists(conn, "main_app", "cooperative"):
|
| 71 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN cooperative TEXT")
|
| 72 |
|
|
if not _column_exists(conn, "main_app", "cooperative_scraped"):
|
| 73 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN cooperative_scraped TEXT")
|
| 74 |
|
|
if not _column_exists(conn, "main_app", "max_players"):
|
| 75 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN max_players TEXT")
|
| 76 |
|
|
if not _column_exists(conn, "main_app", "max_players_scraped"):
|
| 77 |
|
|
conn.execute("ALTER TABLE main_app ADD COLUMN max_players_scraped TEXT")
|
| 78 |
nino.borges |
978 |
|
| 79 |
|
|
# Backfill app_system_id from matched file hashes where possible.
|
| 80 |
|
|
conn.execute(
|
| 81 |
|
|
"""
|
| 82 |
|
|
UPDATE no_intro_main
|
| 83 |
|
|
SET app_system_id = (
|
| 84 |
|
|
SELECT ma.system_console
|
| 85 |
|
|
FROM main_app_file_hash fh
|
| 86 |
|
|
JOIN main_app ma ON ma.id = fh.container_file_id
|
| 87 |
|
|
WHERE lower(fh.file_md5_hash) = lower(no_intro_main.md5)
|
| 88 |
|
|
GROUP BY ma.system_console
|
| 89 |
|
|
ORDER BY COUNT(*) DESC
|
| 90 |
|
|
LIMIT 1
|
| 91 |
|
|
)
|
| 92 |
|
|
WHERE app_system_id IS NULL
|
| 93 |
|
|
"""
|
| 94 |
|
|
)
|
| 95 |
|
|
conn.execute(
|
| 96 |
|
|
"""
|
| 97 |
|
|
UPDATE tosec_main
|
| 98 |
|
|
SET app_system_id = (
|
| 99 |
|
|
SELECT ma.system_console
|
| 100 |
|
|
FROM main_app_file_hash fh
|
| 101 |
|
|
JOIN main_app ma ON ma.id = fh.container_file_id
|
| 102 |
|
|
WHERE lower(fh.file_md5_hash) = lower(tosec_main.md5)
|
| 103 |
|
|
GROUP BY ma.system_console
|
| 104 |
|
|
ORDER BY COUNT(*) DESC
|
| 105 |
|
|
LIMIT 1
|
| 106 |
|
|
)
|
| 107 |
|
|
WHERE app_system_id IS NULL
|
| 108 |
|
|
"""
|
| 109 |
|
|
)
|
| 110 |
|
|
|
| 111 |
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_no_intro_system_md5 ON no_intro_main(app_system_id, md5)")
|
| 112 |
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_tosec_system_md5 ON tosec_main(app_system_id, md5)")
|
| 113 |
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_dat_import_system_source ON dat_import_history(app_system_id, source)")
|
| 114 |
|
|
conn.commit()
|
| 115 |
|
|
|
| 116 |
|
|
|
| 117 |
|
|
def _record_dat_import(conn, app_system_id, source, dat_name, dat_description, dat_version, entry_count):
|
| 118 |
|
|
conn.execute(
|
| 119 |
|
|
"""
|
| 120 |
|
|
INSERT INTO dat_import_history(
|
| 121 |
|
|
app_system_id, source, dat_name, dat_description, dat_version, imported_at, entry_count
|
| 122 |
|
|
) VALUES (?, ?, ?, ?, ?, datetime('now'), ?)
|
| 123 |
|
|
ON CONFLICT(app_system_id, source) DO UPDATE SET
|
| 124 |
|
|
dat_name=excluded.dat_name,
|
| 125 |
|
|
dat_description=excluded.dat_description,
|
| 126 |
|
|
dat_version=excluded.dat_version,
|
| 127 |
|
|
imported_at=excluded.imported_at,
|
| 128 |
|
|
entry_count=excluded.entry_count
|
| 129 |
|
|
""",
|
| 130 |
|
|
(app_system_id, source, dat_name, dat_description, dat_version, entry_count),
|
| 131 |
|
|
)
|
| 132 |
|
|
|
| 133 |
|
|
|
| 134 |
|
|
def ImportNewNoIntroDat(datPath, app_system_id, conn):
|
| 135 |
nino.borges |
976 |
"""Replace No-Intro data using a DAT file."""
|
| 136 |
nino.borges |
978 |
EnsureMetadataSchema(conn)
|
| 137 |
nino.borges |
795 |
tree = ET.parse(datPath)
|
| 138 |
|
|
root = tree.getroot()
|
| 139 |
nino.borges |
976 |
cursor = conn.cursor()
|
| 140 |
nino.borges |
795 |
|
| 141 |
nino.borges |
976 |
header = root.find("header")
|
| 142 |
|
|
if header is None:
|
| 143 |
|
|
raise ValueError("Invalid DAT file: missing <header> section.")
|
| 144 |
nino.borges |
795 |
|
| 145 |
nino.borges |
978 |
headerID = header.findtext("id") or ""
|
| 146 |
|
|
headerName = header.findtext("name") or ""
|
| 147 |
|
|
headerDesc = header.findtext("description") or ""
|
| 148 |
|
|
headerVer = header.findtext("version") or ""
|
| 149 |
nino.borges |
795 |
|
| 150 |
nino.borges |
978 |
cursor.execute("DELETE FROM no_intro_main WHERE app_system_id = ?", (app_system_id,))
|
| 151 |
nino.borges |
976 |
insert_sql = """
|
| 152 |
|
|
INSERT INTO no_intro_main(
|
| 153 |
|
|
game_name, no_intro_id, clone_of_id, description, rom_name,
|
| 154 |
nino.borges |
978 |
crc, md5, sha1, sha256, status, no_intro_system_id, app_system_id
|
| 155 |
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
| 156 |
nino.borges |
976 |
"""
|
| 157 |
nino.borges |
978 |
imported_count = 0
|
| 158 |
nino.borges |
976 |
for elm in root.findall("game"):
|
| 159 |
|
|
gameName = elm.get("name")
|
| 160 |
nino.borges |
978 |
gameID = elm.get("id") or ""
|
| 161 |
nino.borges |
976 |
gameCloneOfID = elm.get("cloneofid")
|
| 162 |
|
|
gameDesc = elm.findtext("description")
|
| 163 |
|
|
|
| 164 |
|
|
rom = elm.find("rom")
|
| 165 |
|
|
if rom is None:
|
| 166 |
|
|
continue
|
| 167 |
|
|
romName = rom.get("name")
|
| 168 |
|
|
romCRC = rom.get("crc")
|
| 169 |
|
|
romMD5 = rom.get("md5")
|
| 170 |
|
|
romSHA1 = rom.get("sha1")
|
| 171 |
|
|
romSHA256 = rom.get("sha256")
|
| 172 |
|
|
romStatus = rom.get("status")
|
| 173 |
|
|
|
| 174 |
|
|
cursor.execute(
|
| 175 |
|
|
insert_sql,
|
| 176 |
|
|
(
|
| 177 |
|
|
gameName,
|
| 178 |
|
|
gameID,
|
| 179 |
|
|
gameCloneOfID,
|
| 180 |
|
|
gameDesc,
|
| 181 |
|
|
romName,
|
| 182 |
|
|
romCRC,
|
| 183 |
|
|
romMD5,
|
| 184 |
|
|
romSHA1,
|
| 185 |
|
|
romSHA256,
|
| 186 |
|
|
romStatus,
|
| 187 |
nino.borges |
978 |
int(headerID) if headerID.isdigit() else 0,
|
| 188 |
|
|
app_system_id,
|
| 189 |
nino.borges |
976 |
),
|
| 190 |
|
|
)
|
| 191 |
nino.borges |
978 |
imported_count += 1
|
| 192 |
nino.borges |
976 |
|
| 193 |
nino.borges |
978 |
_record_dat_import(
|
| 194 |
|
|
conn,
|
| 195 |
|
|
app_system_id,
|
| 196 |
|
|
"no_intro",
|
| 197 |
|
|
headerName,
|
| 198 |
|
|
headerDesc,
|
| 199 |
|
|
headerVer,
|
| 200 |
|
|
imported_count,
|
| 201 |
|
|
)
|
| 202 |
|
|
|
| 203 |
nino.borges |
795 |
conn.commit()
|
| 204 |
|
|
|
| 205 |
|
|
|
| 206 |
nino.borges |
978 |
def ImportNewTosecDat(datPath, app_system_id, conn):
|
| 207 |
nino.borges |
976 |
"""Replace TOSEC data using a DAT file."""
|
| 208 |
nino.borges |
978 |
EnsureMetadataSchema(conn)
|
| 209 |
nino.borges |
907 |
cursor = conn.cursor()
|
| 210 |
nino.borges |
806 |
tree = ET.parse(datPath)
|
| 211 |
|
|
root = tree.getroot()
|
| 212 |
|
|
|
| 213 |
nino.borges |
907 |
header = root.find("header")
|
| 214 |
nino.borges |
976 |
if header is None:
|
| 215 |
|
|
raise ValueError("Invalid DAT file: missing <header> section.")
|
| 216 |
|
|
system_name = header.findtext("name") or ""
|
| 217 |
nino.borges |
978 |
system_desc = header.findtext("description") or ""
|
| 218 |
|
|
version = header.findtext("version") or ""
|
| 219 |
nino.borges |
906 |
|
| 220 |
nino.borges |
978 |
cursor.execute("DELETE FROM tosec_main WHERE app_system_id = ?", (app_system_id,))
|
| 221 |
nino.borges |
976 |
insert_sql = """
|
| 222 |
nino.borges |
978 |
INSERT INTO tosec_main(game_name, description, rom_name, crc, md5, sha1, system_id, app_system_id)
|
| 223 |
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
| 224 |
nino.borges |
976 |
"""
|
| 225 |
nino.borges |
978 |
imported_count = 0
|
| 226 |
nino.borges |
907 |
for game in root.findall("game"):
|
| 227 |
|
|
game_name = game.attrib.get("name", "")
|
| 228 |
nino.borges |
976 |
game_desc = game.findtext("description")
|
| 229 |
nino.borges |
907 |
rom = game.find("rom")
|
| 230 |
|
|
if rom is None:
|
| 231 |
|
|
continue
|
| 232 |
nino.borges |
976 |
cursor.execute(
|
| 233 |
|
|
insert_sql,
|
| 234 |
|
|
(
|
| 235 |
|
|
game_name,
|
| 236 |
|
|
game_desc,
|
| 237 |
|
|
rom.attrib.get("name", ""),
|
| 238 |
|
|
rom.attrib.get("crc", ""),
|
| 239 |
|
|
rom.attrib.get("md5", ""),
|
| 240 |
|
|
rom.attrib.get("sha1", ""),
|
| 241 |
nino.borges |
978 |
app_system_id,
|
| 242 |
|
|
app_system_id,
|
| 243 |
nino.borges |
976 |
),
|
| 244 |
nino.borges |
907 |
)
|
| 245 |
nino.borges |
978 |
imported_count += 1
|
| 246 |
nino.borges |
907 |
|
| 247 |
nino.borges |
978 |
_record_dat_import(
|
| 248 |
|
|
conn,
|
| 249 |
|
|
app_system_id,
|
| 250 |
|
|
"tosec",
|
| 251 |
|
|
system_name,
|
| 252 |
|
|
system_desc,
|
| 253 |
|
|
version,
|
| 254 |
|
|
imported_count,
|
| 255 |
|
|
)
|
| 256 |
|
|
|
| 257 |
nino.borges |
806 |
conn.commit()
|
| 258 |
nino.borges |
907 |
print("TOSEC DAT imported successfully.")
|
| 259 |
nino.borges |
806 |
|
| 260 |
nino.borges |
906 |
|
| 261 |
nino.borges |
907 |
def AddNewSystem(systemName, shortName, relativePath, conn):
|
| 262 |
nino.borges |
805 |
"""Adding a single system, with path where the ROMs will live."""
|
| 263 |
nino.borges |
976 |
conn.execute(
|
| 264 |
|
|
'INSERT INTO main_app_system("name", "short_name", "relative_file_path") VALUES (?, ?, ?)',
|
| 265 |
|
|
(systemName, shortName, relativePath),
|
| 266 |
|
|
)
|
| 267 |
nino.borges |
805 |
conn.commit()
|
| 268 |
|
|
|
| 269 |
nino.borges |
976 |
|
| 270 |
nino.borges |
907 |
def AddNewRoms(pathToRoms, systemID, conn, testOnly = False):
|
| 271 |
nino.borges |
795 |
"""Adding additional roms."""
|
| 272 |
nino.borges |
805 |
importedCount = 0
|
| 273 |
|
|
alreadyExistsCount = 0
|
| 274 |
|
|
errImportCount = 0
|
| 275 |
|
|
|
| 276 |
nino.borges |
976 |
system_row = conn.execute(
|
| 277 |
|
|
"SELECT relative_file_path FROM main_app_system WHERE id = ?",
|
| 278 |
|
|
(systemID,),
|
| 279 |
|
|
).fetchone()
|
| 280 |
|
|
if not system_row:
|
| 281 |
|
|
raise ValueError(f"System id {systemID} not found in main_app_system.")
|
| 282 |
|
|
|
| 283 |
|
|
systemRomPath = system_row[0]
|
| 284 |
|
|
fullRomPath = os.path.join(config.get("RootConfig", "softwareBackupStartDir"), systemRomPath)
|
| 285 |
|
|
os.makedirs(fullRomPath, exist_ok=True)
|
| 286 |
|
|
|
| 287 |
|
|
currentHashList = {
|
| 288 |
|
|
row[0]
|
| 289 |
|
|
for row in conn.execute("SELECT container_md5_hash FROM main_app WHERE container_md5_hash IS NOT NULL")
|
| 290 |
|
|
}
|
| 291 |
nino.borges |
795 |
print(len(currentHashList))
|
| 292 |
nino.borges |
805 |
|
| 293 |
nino.borges |
976 |
hshr = Tool_Box.NinoGenTools.HashFileContents("md5")
|
| 294 |
nino.borges |
795 |
for testFile in os.listdir(pathToRoms):
|
| 295 |
nino.borges |
976 |
source_path = os.path.join(pathToRoms, testFile)
|
| 296 |
|
|
if not os.path.isfile(source_path):
|
| 297 |
|
|
continue
|
| 298 |
|
|
|
| 299 |
nino.borges |
805 |
print(f"\n\nNow analyzing {testFile}...")
|
| 300 |
nino.borges |
976 |
hashVal = hshr.HashFile(source_path)
|
| 301 |
nino.borges |
795 |
if hashVal in currentHashList:
|
| 302 |
|
|
print("This file is already in your collection, skipping.")
|
| 303 |
nino.borges |
805 |
alreadyExistsCount +=1
|
| 304 |
nino.borges |
795 |
else:
|
| 305 |
|
|
print("This file is unique. Adding to collection.")
|
| 306 |
nino.borges |
806 |
fileNameInc = 0
|
| 307 |
|
|
targetFileName = testFile
|
| 308 |
|
|
while os.path.isfile(os.path.join(fullRomPath,targetFileName)):
|
| 309 |
|
|
fileNameInc +=1
|
| 310 |
|
|
testFileNamePart, testFileExt = os.path.splitext(testFile)
|
| 311 |
|
|
targetFileName = testFileNamePart +"_"+ str(fileNameInc) + testFileExt
|
| 312 |
|
|
if testOnly:
|
| 313 |
|
|
pass
|
| 314 |
nino.borges |
805 |
else:
|
| 315 |
nino.borges |
976 |
try:
|
| 316 |
|
|
shutil.copyfile(source_path, os.path.join(fullRomPath, targetFileName))
|
| 317 |
|
|
conn.execute(
|
| 318 |
|
|
'INSERT INTO main_app("container_file_name", "system_console", "container_md5_hash") VALUES(?, ?, ?)',
|
| 319 |
|
|
(targetFileName, systemID, hashVal),
|
| 320 |
|
|
)
|
| 321 |
|
|
conn.commit()
|
| 322 |
|
|
except Exception:
|
| 323 |
|
|
errImportCount += 1
|
| 324 |
|
|
continue
|
| 325 |
|
|
currentHashList.add(hashVal)
|
| 326 |
nino.borges |
806 |
importedCount +=1
|
| 327 |
nino.borges |
805 |
return importedCount, alreadyExistsCount, errImportCount
|
| 328 |
nino.borges |
795 |
|
| 329 |
nino.borges |
976 |
|
| 330 |
nino.borges |
979 |
def _normalize_match_key(name):
|
| 331 |
|
|
if not name:
|
| 332 |
|
|
return ""
|
| 333 |
|
|
normalized = name.lower().strip()
|
| 334 |
|
|
# Normalize common filename separators to improve filename/title matching.
|
| 335 |
|
|
for ch in ("_", "-", ".", ",", "!", ":", ";", "'", '"', "(", ")", "[", "]"):
|
| 336 |
|
|
normalized = normalized.replace(ch, " ")
|
| 337 |
|
|
return " ".join(normalized.split())
|
| 338 |
|
|
|
| 339 |
|
|
|
| 340 |
|
|
def GetSystemListWithRoms(conn):
|
| 341 |
|
|
res = conn.execute(
|
| 342 |
|
|
"""
|
| 343 |
|
|
SELECT s.id, s.name, s.short_name, s.relative_file_path
|
| 344 |
|
|
FROM main_app_system s
|
| 345 |
|
|
WHERE EXISTS (
|
| 346 |
|
|
SELECT 1 FROM main_app ma WHERE ma.system_console = s.id
|
| 347 |
|
|
)
|
| 348 |
|
|
ORDER BY s.name
|
| 349 |
|
|
"""
|
| 350 |
|
|
)
|
| 351 |
|
|
systemNamesMatrix = {}
|
| 352 |
|
|
for h in res:
|
| 353 |
|
|
systemNamesMatrix[h[1]] = [h[0], h[2], h[3]]
|
| 354 |
|
|
return systemNamesMatrix
|
| 355 |
|
|
|
| 356 |
|
|
|
| 357 |
|
|
def AddNewArtwork(pathToArtwork, systemID, artworkType, conn, testOnly=False):
|
| 358 |
|
|
"""
|
| 359 |
|
|
Ingest artwork for a system by matching names in this order:
|
| 360 |
|
|
game_name -> no_intro_game_name -> tosec_game_name -> filename stem.
|
| 361 |
|
|
Only updates rows where the target artwork field is empty.
|
| 362 |
|
|
"""
|
| 363 |
|
|
artwork_field_map = {
|
| 364 |
|
|
"box": "path_to_screenshot_box",
|
| 365 |
|
|
"title": "path_to_screenshot_title",
|
| 366 |
|
|
"ingame": "path_to_screenshot_ingame",
|
| 367 |
|
|
}
|
| 368 |
|
|
if artworkType not in artwork_field_map:
|
| 369 |
|
|
raise ValueError(f"Unsupported artwork type: {artworkType}")
|
| 370 |
|
|
|
| 371 |
|
|
target_field = artwork_field_map[artworkType]
|
| 372 |
|
|
system_row = conn.execute(
|
| 373 |
|
|
"SELECT short_name FROM main_app_system WHERE id = ?",
|
| 374 |
|
|
(systemID,),
|
| 375 |
|
|
).fetchone()
|
| 376 |
|
|
if not system_row:
|
| 377 |
|
|
raise ValueError(f"System id {systemID} not found.")
|
| 378 |
|
|
system_short_name = system_row[0]
|
| 379 |
|
|
|
| 380 |
|
|
artwork_root = config.get("RootConfig", "softwareArtworkDir")
|
| 381 |
|
|
target_dir = os.path.join(artwork_root, system_short_name)
|
| 382 |
|
|
os.makedirs(target_dir, exist_ok=True)
|
| 383 |
|
|
|
| 384 |
|
|
source_file_map = {}
|
| 385 |
|
|
for entry in os.listdir(pathToArtwork):
|
| 386 |
|
|
full_path = os.path.join(pathToArtwork, entry)
|
| 387 |
|
|
if not os.path.isfile(full_path):
|
| 388 |
|
|
continue
|
| 389 |
|
|
stem, _ext = os.path.splitext(entry)
|
| 390 |
|
|
key = _normalize_match_key(stem)
|
| 391 |
|
|
if key and key not in source_file_map:
|
| 392 |
|
|
source_file_map[key] = full_path
|
| 393 |
|
|
|
| 394 |
|
|
no_intro_scoped = _column_exists(conn, "no_intro_main", "app_system_id")
|
| 395 |
|
|
tosec_scoped = _column_exists(conn, "tosec_main", "app_system_id")
|
| 396 |
|
|
no_intro_join = (
|
| 397 |
|
|
"LEFT JOIN no_intro_main nim ON lower(fh.file_md5_hash)=lower(nim.md5) AND nim.app_system_id=ma.system_console"
|
| 398 |
|
|
if no_intro_scoped
|
| 399 |
|
|
else "LEFT JOIN no_intro_main nim ON lower(fh.file_md5_hash)=lower(nim.md5)"
|
| 400 |
|
|
)
|
| 401 |
|
|
tosec_join = (
|
| 402 |
|
|
"LEFT JOIN tosec_main tm ON lower(fh.file_md5_hash)=lower(tm.md5) AND tm.app_system_id=ma.system_console"
|
| 403 |
|
|
if tosec_scoped
|
| 404 |
|
|
else "LEFT JOIN tosec_main tm ON lower(fh.file_md5_hash)=lower(tm.md5)"
|
| 405 |
|
|
)
|
| 406 |
|
|
|
| 407 |
|
|
rows = conn.execute(
|
| 408 |
|
|
f"""
|
| 409 |
|
|
SELECT ma.id,
|
| 410 |
|
|
ma.game_name,
|
| 411 |
|
|
nim.game_name,
|
| 412 |
|
|
tm.game_name,
|
| 413 |
|
|
COALESCE(fh.file_name, ma.container_file_name) AS best_filename,
|
| 414 |
|
|
ma.{target_field}
|
| 415 |
|
|
FROM main_app ma
|
| 416 |
|
|
LEFT JOIN main_app_file_hash fh
|
| 417 |
|
|
ON fh.id = (
|
| 418 |
|
|
SELECT MIN(fh2.id)
|
| 419 |
|
|
FROM main_app_file_hash fh2
|
| 420 |
|
|
WHERE fh2.container_file_id = ma.id
|
| 421 |
|
|
)
|
| 422 |
|
|
{no_intro_join}
|
| 423 |
|
|
{tosec_join}
|
| 424 |
|
|
WHERE ma.system_console = ?
|
| 425 |
|
|
""",
|
| 426 |
|
|
(systemID,),
|
| 427 |
|
|
)
|
| 428 |
|
|
|
| 429 |
|
|
matched_count = 0
|
| 430 |
|
|
skipped_existing_count = 0
|
| 431 |
|
|
unmatched_count = 0
|
| 432 |
|
|
error_count = 0
|
| 433 |
|
|
|
| 434 |
|
|
for game_id, game_name, no_intro_name, tosec_name, best_filename, existing_path in rows:
|
| 435 |
|
|
if existing_path:
|
| 436 |
|
|
skipped_existing_count += 1
|
| 437 |
|
|
continue
|
| 438 |
|
|
|
| 439 |
|
|
filename_stem = os.path.splitext(best_filename)[0] if best_filename else ""
|
| 440 |
|
|
candidate_names = [game_name, no_intro_name, tosec_name, filename_stem]
|
| 441 |
|
|
|
| 442 |
|
|
source_match = None
|
| 443 |
|
|
for candidate in candidate_names:
|
| 444 |
|
|
key = _normalize_match_key(candidate)
|
| 445 |
|
|
if key and key in source_file_map:
|
| 446 |
|
|
source_match = source_file_map[key]
|
| 447 |
|
|
break
|
| 448 |
|
|
|
| 449 |
|
|
if not source_match:
|
| 450 |
|
|
unmatched_count += 1
|
| 451 |
|
|
continue
|
| 452 |
|
|
|
| 453 |
|
|
source_name = os.path.basename(source_match)
|
| 454 |
|
|
source_stem, source_ext = os.path.splitext(source_name)
|
| 455 |
|
|
target_name = f"{artworkType}_{source_stem}{source_ext}"
|
| 456 |
|
|
target_path = os.path.join(target_dir, target_name)
|
| 457 |
|
|
|
| 458 |
|
|
increment = 1
|
| 459 |
|
|
while os.path.exists(target_path):
|
| 460 |
|
|
target_name = f"{artworkType}_{source_stem}_{increment}{source_ext}"
|
| 461 |
|
|
target_path = os.path.join(target_dir, target_name)
|
| 462 |
|
|
increment += 1
|
| 463 |
|
|
|
| 464 |
|
|
relative_path = os.path.relpath(target_path, artwork_root)
|
| 465 |
|
|
if testOnly:
|
| 466 |
|
|
matched_count += 1
|
| 467 |
|
|
continue
|
| 468 |
|
|
|
| 469 |
|
|
try:
|
| 470 |
|
|
shutil.copyfile(source_match, target_path)
|
| 471 |
|
|
conn.execute(
|
| 472 |
|
|
f"UPDATE main_app SET {target_field} = ? WHERE id = ?",
|
| 473 |
|
|
(relative_path, game_id),
|
| 474 |
|
|
)
|
| 475 |
|
|
matched_count += 1
|
| 476 |
|
|
except Exception:
|
| 477 |
|
|
error_count += 1
|
| 478 |
|
|
|
| 479 |
|
|
if not testOnly:
|
| 480 |
|
|
conn.commit()
|
| 481 |
|
|
|
| 482 |
|
|
return matched_count, skipped_existing_count, unmatched_count, error_count
|
| 483 |
|
|
|
| 484 |
|
|
|
| 485 |
nino.borges |
907 |
def GetSystemList(conn):
|
| 486 |
nino.borges |
805 |
res = conn.execute("SELECT id, name, short_name, relative_file_path FROM main_app_system ")
|
| 487 |
|
|
systemNamesMatrix = {}
|
| 488 |
|
|
for h in res:
|
| 489 |
|
|
systemNamesMatrix[h[1]] = [h[0],h[2],h[3]]
|
| 490 |
|
|
return systemNamesMatrix
|
| 491 |
|
|
|
| 492 |
nino.borges |
907 |
def GetGameListBySystem(systemName, conn):
|
| 493 |
nino.borges |
976 |
system_row = conn.execute(
|
| 494 |
|
|
"""
|
| 495 |
|
|
SELECT id
|
| 496 |
|
|
FROM main_app_system
|
| 497 |
|
|
WHERE lower(short_name) = lower(?)
|
| 498 |
|
|
OR lower(name) = lower(?)
|
| 499 |
|
|
""",
|
| 500 |
|
|
(systemName, systemName),
|
| 501 |
|
|
).fetchone()
|
| 502 |
|
|
if not system_row:
|
| 503 |
|
|
return {}
|
| 504 |
|
|
|
| 505 |
|
|
system_id = system_row[0]
|
| 506 |
nino.borges |
978 |
no_intro_scoped = _column_exists(conn, "no_intro_main", "app_system_id")
|
| 507 |
|
|
tosec_scoped = _column_exists(conn, "tosec_main", "app_system_id")
|
| 508 |
|
|
|
| 509 |
|
|
no_intro_join = (
|
| 510 |
|
|
"LEFT JOIN no_intro_main nim ON lower(fh.file_md5_hash) = lower(nim.md5) "
|
| 511 |
|
|
"AND nim.app_system_id = ma.system_console"
|
| 512 |
|
|
if no_intro_scoped
|
| 513 |
|
|
else "LEFT JOIN no_intro_main nim ON lower(fh.file_md5_hash) = lower(nim.md5)"
|
| 514 |
|
|
)
|
| 515 |
|
|
tosec_join = (
|
| 516 |
|
|
"LEFT JOIN tosec_main tm ON lower(fh.file_md5_hash) = lower(tm.md5) "
|
| 517 |
|
|
"AND tm.app_system_id = ma.system_console"
|
| 518 |
|
|
if tosec_scoped
|
| 519 |
|
|
else "LEFT JOIN tosec_main tm ON lower(fh.file_md5_hash) = lower(tm.md5)"
|
| 520 |
|
|
)
|
| 521 |
|
|
|
| 522 |
nino.borges |
976 |
res = conn.execute(
|
| 523 |
nino.borges |
978 |
f"""
|
| 524 |
nino.borges |
976 |
SELECT ma.id,
|
| 525 |
|
|
ma.game_name,
|
| 526 |
|
|
nim.game_name,
|
| 527 |
|
|
tm.game_name,
|
| 528 |
|
|
COALESCE(fh.file_name, ma.container_file_name)
|
| 529 |
|
|
FROM main_app ma
|
| 530 |
|
|
LEFT JOIN main_app_file_hash fh
|
| 531 |
|
|
ON fh.id = (
|
| 532 |
|
|
SELECT MIN(fh2.id)
|
| 533 |
|
|
FROM main_app_file_hash fh2
|
| 534 |
|
|
WHERE fh2.container_file_id = ma.id
|
| 535 |
|
|
)
|
| 536 |
nino.borges |
978 |
{no_intro_join}
|
| 537 |
|
|
{tosec_join}
|
| 538 |
nino.borges |
976 |
WHERE ma.system_console = ?
|
| 539 |
|
|
""",
|
| 540 |
|
|
(system_id,),
|
| 541 |
|
|
)
|
| 542 |
nino.borges |
795 |
gameNamesMatrix = {}
|
| 543 |
nino.borges |
976 |
duplicate_counter = {}
|
| 544 |
nino.borges |
795 |
for h in res:
|
| 545 |
|
|
if h[1]:
|
| 546 |
nino.borges |
976 |
display_name = h[1]
|
| 547 |
nino.borges |
795 |
elif h[2]:
|
| 548 |
nino.borges |
976 |
display_name = h[2]
|
| 549 |
nino.borges |
806 |
elif h[3]:
|
| 550 |
nino.borges |
976 |
display_name = h[3]
|
| 551 |
nino.borges |
795 |
else:
|
| 552 |
nino.borges |
976 |
display_name = h[4] or f"Unknown-{h[0]}"
|
| 553 |
nino.borges |
795 |
|
| 554 |
nino.borges |
976 |
# Keep titles unique in the listbox mapping.
|
| 555 |
|
|
if display_name in gameNamesMatrix:
|
| 556 |
|
|
duplicate_counter[display_name] = duplicate_counter.get(display_name, 1) + 1
|
| 557 |
|
|
unique_name = f"{display_name} ({duplicate_counter[display_name]})"
|
| 558 |
|
|
gameNamesMatrix[unique_name] = h[0]
|
| 559 |
|
|
else:
|
| 560 |
|
|
duplicate_counter[display_name] = 1
|
| 561 |
|
|
gameNamesMatrix[display_name] = h[0]
|
| 562 |
|
|
|
| 563 |
nino.borges |
795 |
return gameNamesMatrix
|
| 564 |
|
|
|
| 565 |
nino.borges |
976 |
|
| 566 |
nino.borges |
907 |
def GetSingleGameById(game_id, conn):
|
| 567 |
|
|
"""Fetches full metadata for a single ROM entry, including No-Intro and TOSEC info if available."""
|
| 568 |
|
|
cursor = conn.cursor()
|
| 569 |
|
|
|
| 570 |
|
|
cursor.execute("""
|
| 571 |
nino.borges |
976 |
SELECT ma.id,
|
| 572 |
|
|
ma.game_name,
|
| 573 |
|
|
ma.container_file_name,
|
| 574 |
|
|
sys.name,
|
| 575 |
|
|
sys.short_name,
|
| 576 |
|
|
sys.relative_file_path,
|
| 577 |
nino.borges |
978 |
ma.container_md5_hash,
|
| 578 |
nino.borges |
979 |
ma.system_console,
|
| 579 |
|
|
ma.path_to_screenshot_box,
|
| 580 |
|
|
ma.path_to_screenshot_title,
|
| 581 |
nino.borges |
990 |
ma.path_to_screenshot_ingame,
|
| 582 |
|
|
ma.favorite_game,
|
| 583 |
|
|
ma.release_date,
|
| 584 |
|
|
ma.release_date_scraped,
|
| 585 |
|
|
ma.game_genre,
|
| 586 |
|
|
ma.game_genre_scraped,
|
| 587 |
|
|
ma.cooperative,
|
| 588 |
|
|
ma.cooperative_scraped,
|
| 589 |
|
|
ma.max_players,
|
| 590 |
|
|
ma.max_players_scraped,
|
| 591 |
|
|
ma.description,
|
| 592 |
|
|
ma.description_scraped,
|
| 593 |
|
|
ma.user_notes
|
| 594 |
nino.borges |
907 |
FROM main_app ma
|
| 595 |
nino.borges |
976 |
LEFT JOIN main_app_system sys ON ma.system_console = sys.id
|
| 596 |
nino.borges |
907 |
WHERE ma.id = ?
|
| 597 |
|
|
""", (game_id,))
|
| 598 |
|
|
game = cursor.fetchone()
|
| 599 |
nino.borges |
976 |
if not game:
|
| 600 |
|
|
return None
|
| 601 |
nino.borges |
907 |
|
| 602 |
nino.borges |
976 |
file_hash_row = cursor.execute(
|
| 603 |
|
|
"""
|
| 604 |
|
|
SELECT file_name, file_md5_hash
|
| 605 |
|
|
FROM main_app_file_hash
|
| 606 |
|
|
WHERE container_file_id = ?
|
| 607 |
|
|
ORDER BY id ASC
|
| 608 |
|
|
LIMIT 1
|
| 609 |
|
|
""",
|
| 610 |
|
|
(game[0],),
|
| 611 |
|
|
).fetchone()
|
| 612 |
|
|
|
| 613 |
|
|
resolved_file_name = file_hash_row[0] if file_hash_row and file_hash_row[0] else game[2]
|
| 614 |
|
|
resolved_hash = file_hash_row[1] if file_hash_row and file_hash_row[1] else game[6]
|
| 615 |
|
|
|
| 616 |
nino.borges |
978 |
no_intro_scoped = _column_exists(conn, "no_intro_main", "app_system_id")
|
| 617 |
|
|
if no_intro_scoped:
|
| 618 |
|
|
no_intro = cursor.execute(
|
| 619 |
|
|
"""
|
| 620 |
|
|
SELECT nim.game_name
|
| 621 |
|
|
FROM no_intro_main nim
|
| 622 |
|
|
WHERE lower(nim.md5) = lower(?)
|
| 623 |
|
|
AND nim.app_system_id = ?
|
| 624 |
|
|
LIMIT 1
|
| 625 |
|
|
""",
|
| 626 |
|
|
(resolved_hash, game[7]),
|
| 627 |
|
|
).fetchone()
|
| 628 |
|
|
else:
|
| 629 |
|
|
no_intro = cursor.execute(
|
| 630 |
|
|
"""
|
| 631 |
|
|
SELECT nim.game_name
|
| 632 |
|
|
FROM no_intro_main nim
|
| 633 |
|
|
WHERE lower(nim.md5) = lower(?)
|
| 634 |
|
|
LIMIT 1
|
| 635 |
|
|
""",
|
| 636 |
|
|
(resolved_hash,),
|
| 637 |
|
|
).fetchone()
|
| 638 |
nino.borges |
907 |
|
| 639 |
nino.borges |
978 |
tosec_scoped = _column_exists(conn, "tosec_main", "app_system_id")
|
| 640 |
|
|
if tosec_scoped:
|
| 641 |
|
|
tosec = cursor.execute(
|
| 642 |
|
|
"""
|
| 643 |
|
|
SELECT tm.game_name
|
| 644 |
|
|
FROM tosec_main tm
|
| 645 |
|
|
WHERE lower(tm.md5) = lower(?)
|
| 646 |
|
|
AND tm.app_system_id = ?
|
| 647 |
|
|
LIMIT 1
|
| 648 |
|
|
""",
|
| 649 |
|
|
(resolved_hash, game[7]),
|
| 650 |
|
|
).fetchone()
|
| 651 |
|
|
else:
|
| 652 |
|
|
tosec = cursor.execute(
|
| 653 |
|
|
"""
|
| 654 |
|
|
SELECT tm.game_name
|
| 655 |
|
|
FROM tosec_main tm
|
| 656 |
|
|
WHERE lower(tm.md5) = lower(?)
|
| 657 |
|
|
LIMIT 1
|
| 658 |
|
|
""",
|
| 659 |
|
|
(resolved_hash,),
|
| 660 |
|
|
).fetchone()
|
| 661 |
nino.borges |
907 |
|
| 662 |
nino.borges |
976 |
if game[5]:
|
| 663 |
|
|
file_path = os.path.join(
|
| 664 |
|
|
config.get("RootConfig", "softwareBackupStartDir"),
|
| 665 |
|
|
game[5],
|
| 666 |
|
|
resolved_file_name or "",
|
| 667 |
|
|
)
|
| 668 |
|
|
else:
|
| 669 |
|
|
file_path = resolved_file_name or ""
|
| 670 |
nino.borges |
907 |
|
| 671 |
nino.borges |
979 |
artwork_root = config.get("RootConfig", "softwareArtworkDir")
|
| 672 |
|
|
|
| 673 |
|
|
def resolve_artwork_path(relative_or_absolute):
|
| 674 |
|
|
if not relative_or_absolute:
|
| 675 |
|
|
return None
|
| 676 |
|
|
if os.path.isabs(relative_or_absolute):
|
| 677 |
|
|
return relative_or_absolute
|
| 678 |
|
|
return os.path.join(artwork_root, relative_or_absolute)
|
| 679 |
|
|
|
| 680 |
nino.borges |
907 |
return {
|
| 681 |
|
|
"id": game[0],
|
| 682 |
nino.borges |
976 |
"game_name": game[1],
|
| 683 |
|
|
"hash": resolved_hash,
|
| 684 |
|
|
"filename": resolved_file_name,
|
| 685 |
|
|
"path": file_path,
|
| 686 |
|
|
"system_name": game[3],
|
| 687 |
|
|
"system_short_name": game[4],
|
| 688 |
nino.borges |
978 |
"no_intro_system": game[3],
|
| 689 |
|
|
"no_intro_game": no_intro[0] if no_intro else None,
|
| 690 |
|
|
"tosec_system": game[3],
|
| 691 |
|
|
"tosec_game": tosec[0] if tosec else None,
|
| 692 |
nino.borges |
979 |
"box_art_path": resolve_artwork_path(game[8]),
|
| 693 |
|
|
"title_art_path": resolve_artwork_path(game[9]),
|
| 694 |
|
|
"ingame_art_path": resolve_artwork_path(game[10]),
|
| 695 |
nino.borges |
990 |
"favorite_game": game[11],
|
| 696 |
|
|
"release_date": game[12],
|
| 697 |
|
|
"release_date_scraped": game[13],
|
| 698 |
|
|
"game_genre": game[14],
|
| 699 |
|
|
"game_genre_scraped": game[15],
|
| 700 |
|
|
"cooperative": game[16],
|
| 701 |
|
|
"cooperative_scraped": game[17],
|
| 702 |
|
|
"max_players": game[18],
|
| 703 |
|
|
"max_players_scraped": game[19],
|
| 704 |
|
|
"description": game[20],
|
| 705 |
|
|
"description_scraped": game[21],
|
| 706 |
|
|
"user_notes": game[22],
|
| 707 |
nino.borges |
907 |
}
|
| 708 |
|
|
|
| 709 |
nino.borges |
990 |
|
| 710 |
|
|
def SaveGameUserProperties(game_id, values, conn):
|
| 711 |
|
|
"""Persist user-controlled game properties for a selected game."""
|
| 712 |
|
|
conn.execute(
|
| 713 |
|
|
"""
|
| 714 |
|
|
UPDATE main_app
|
| 715 |
|
|
SET favorite_game = ?,
|
| 716 |
|
|
release_date = ?,
|
| 717 |
|
|
game_genre = ?,
|
| 718 |
|
|
cooperative = ?,
|
| 719 |
|
|
max_players = ?,
|
| 720 |
|
|
description = ?,
|
| 721 |
|
|
user_notes = ?
|
| 722 |
|
|
WHERE id = ?
|
| 723 |
|
|
""",
|
| 724 |
|
|
(
|
| 725 |
|
|
values.get("favorite_game"),
|
| 726 |
|
|
values.get("release_date"),
|
| 727 |
|
|
values.get("game_genre"),
|
| 728 |
|
|
values.get("cooperative"),
|
| 729 |
|
|
values.get("max_players"),
|
| 730 |
|
|
values.get("description"),
|
| 731 |
|
|
values.get("user_notes"),
|
| 732 |
|
|
game_id,
|
| 733 |
|
|
),
|
| 734 |
|
|
)
|
| 735 |
|
|
conn.commit()
|
| 736 |
|
|
|
| 737 |
nino.borges |
795 |
|
| 738 |
|
|
|
| 739 |
|
|
|
| 740 |
|
|
if __name__ == '__main__':
|
| 741 |
nino.borges |
806 |
pass
|
| 742 |
nino.borges |
805 |
#test = GetSystemList()
|
| 743 |
|
|
#print(test.keys())
|
| 744 |
|
|
#AddNewSystem("Super Nintendo", "SNES", r"Roms/Snes/Gromulus")
|
| 745 |
nino.borges |
795 |
#ImportNewNoIntroDat("/home/nino/MyCode/Python/Active_prgs/Gromulus/_data/_dats/Nintendo - Super Nintendo Entertainment System (20230516-033614).dat")
|
| 746 |
nino.borges |
806 |
#AddNewRoms('/mnt/smb/HomeData/ninoborges/Emulation/Roms/Snes/Random2',"1",testOnly=True)
|
| 747 |
nino.borges |
795 |
#res = conn.execute("SELECT * FROM main_app LEFT JOIN no_intro_main ON main_app.md5 = no_intro_main.md5")
|
| 748 |
|
|
#for h in res:
|
| 749 |
|
|
# print(h)
|
| 750 |
|
|
# print("\n")
|
| 751 |
|
|
#GetGameListBySystem('snes')
|
| 752 |
nino.borges |
805 |
#GetSingleGameById('224')
|
| 753 |
nino.borges |
991 |
|
| 754 |
|
|
|