From e62d74dd06a05e69f9304f8ec08955b12b6d93d5 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sat, 2 Nov 2024 20:49:38 -0400 Subject: [PATCH 01/30] first version of game importer queue screen - not hooked up to actual imports This commit adds the import status screen and connects it to a not-yet-used queue to manage the state of imports. --- .../Services/GameImporter/GameImporter.swift | 135 ++++++++++++++++++ .../PVSwiftUI/Imports/ImportStatusView.swift | 83 +++++++++++ .../PVSwiftUI/RootView/PVMenuDelegate.swift | 35 ++++- .../PVSwiftUI/SideMenu/SideMenuView.swift | 4 + .../PVGameLibraryUpdatesController.swift | 3 +- 5 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index e15dde8dfb..dcc1dcc111 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -23,6 +23,7 @@ import PVLogging import PVPrimitives import PVRealm import Perception +import SwiftUI #if canImport(UIKit) import UIKit @@ -101,6 +102,67 @@ public typealias GameImporterFinishedImportingGameHandler = (_ md5Hash: String, /// Type alias for a closure that handles the finish of getting artwork public typealias GameImporterFinishedGettingArtworkHandler = (_ artworkURL: String?) -> Void + +// Enum to define the possible statuses of each import +public enum ImportStatus: String { + case queued + case processing + case success + case failure + case conflict // Indicates additional action needed by user after successful import + + public var description: String { + switch self { + case .queued: return "Queued" + case .processing: return "Processing" + case .success: return "Completed" + case .failure: return "Failed" + case .conflict: return "Conflict" + } + } + + public var color: Color { + switch self { + case .queued: return .gray + case .processing: return .blue + case .success: return .green + case .failure: return .red + case .conflict: return .yellow + } + } +} + +// Enum to define file types for each import +public enum FileType { + case bios, artwork, game, cdRom, unknown +} + +// Enum to track processing state +public enum ProcessingState { + case idle + case processing +} + +// ImportItem model to hold each file's metadata and progress +@Observable +public class ImportItem: Identifiable, ObservableObject { + public let id = UUID() + public let url: URL + public let fileType: FileType + public let system: String // Can be set to the specific system type + + // Observable status for individual imports + public var status: ImportStatus = .queued + + public init(url: URL, fileType: FileType, system: String) { + self.url = url + self.fileType = fileType + self.system = system + } +} + + + #if !os(tvOS) @Observable #else @@ -151,6 +213,79 @@ public final class GameImporter: ObservableObject { /// Map of ROM extensions to their corresponding system identifiers public private(set) var romExtensionToSystemsMap = [String: [String]]() + // MARK: - Queue + + public var importStatus: String = "" + + public var importQueue: [ImportItem] = [] + + public var processingState: ProcessingState = .idle // Observable state for processing status + + // Adds an ImportItem to the queue without starting processing + public func addImport(_ item: ImportItem) { + importQueue.append(item) + + startProcessing() + } + + // Public method to manually start processing if needed + public func startProcessing() { + // Only start processing if it's not already active + guard processingState == .idle else { return } + Task { + await processQueue() + } + } + + // Processes each ImportItem in the queue sequentially + private func processQueue() async { + DispatchQueue.main.async { + self.processingState = .processing + } + + for item in importQueue where item.status == .queued { + await processItem(item) + } + + DispatchQueue.main.async { + self.processingState = .idle // Reset processing status when queue is fully processed + } + } + + // Process a single ImportItem and update its status + private func processItem(_ item: ImportItem) async { + item.status = .processing + updateStatus("Importing \(item.url.lastPathComponent) for \(item.system)") + + do { + // Simulate file processing + try await performImport(for: item) + item.status = .success + updateStatus("Completed \(item.url.lastPathComponent) for \(item.system)") + } catch { + if error.localizedDescription.contains("Conflict") { + item.status = .conflict + updateStatus("Conflict for \(item.url.lastPathComponent). User action needed.") + } else { + item.status = .failure + updateStatus("Failed \(item.url.lastPathComponent) with error: \(error.localizedDescription)") + } + } + } + + private func performImport(for item: ImportItem) async throws { + try await Task.sleep(nanoseconds: 5_000_000_000) + } + + // General status update for GameImporter + private func updateStatus(_ message: String) { + DispatchQueue.main.async { + self.importStatus = message + } + } + + + // MARK: - Paths /// Path to the documents directory diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift new file mode 100644 index 0000000000..c5d4bed30a --- /dev/null +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -0,0 +1,83 @@ + +// +// ImportStatusView.swift +// PVUI +// +// Created by David Proskin on 10/31/24. +// + +import SwiftUI +import PVLibrary + +public protocol ImportStatusDelegate : AnyObject { + func dismissAction() + func addImportsAction() +} + +// View Model to manage import tasks +class ImportViewModel: ObservableObject { + public let gameImporter = GameImporter.shared +} + +// Individual Import Task Row View +struct ImportTaskRowView: View { + let item: ImportItem + + var body: some View { + HStack { + VStack(alignment: .leading) { + Text(item.url.lastPathComponent) + .font(.headline) + Text(item.status.description) + .font(.subheadline) + .foregroundColor(item.status.color) + } + + Spacer() + + if item.status == .processing { + ProgressView().progressViewStyle(.circular).frame(width: 40, height: 40, alignment: .center) + } else { + Image(systemName: item.status == .success ? "checkmark.circle.fill" : "xmark.circle.fill") + .foregroundColor(item.status.color) + } + } + .padding() + .background(Color.white) + .cornerRadius(10) + .shadow(color: .gray.opacity(0.2), radius: 5, x: 0, y: 2) + } +} + +struct ImportStatusView: View { + @ObservedObject var updatesController: PVGameLibraryUpdatesController + var viewModel:ImportViewModel + weak var delegate:ImportStatusDelegate! + + var body: some View { + NavigationView { + ScrollView { + LazyVStack(spacing: 10) { + ForEach(viewModel.gameImporter.importQueue) { item in + ImportTaskRowView(item: item).id(item.id) + } + } + .padding() + } + .navigationTitle("Import Status") + .navigationBarItems( + leading: Button("Done") { delegate.dismissAction() }, + trailing: + Button("Import Files") { + delegate?.addImportsAction() + } + ) + } + .presentationDetents([.medium, .large]) + .presentationDragIndicator(.visible) + } +} + +#Preview { + +} diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift index 53fd11f95f..df10e5f983 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift @@ -21,6 +21,7 @@ import PVRealm import PVPrimitives import PVLogging import UniformTypeIdentifiers + #if canImport(FreemiumKit) import FreemiumKit #endif @@ -35,6 +36,7 @@ import SafariServices // MARK: - Menu Delegate public protocol PVMenuDelegate: AnyObject { + func didTapImports() func didTapSettings() func didTapHome() func didTapAddGames() @@ -44,6 +46,17 @@ public protocol PVMenuDelegate: AnyObject { @available(iOS 14, tvOS 14, *) extension PVRootViewController: PVMenuDelegate { + + public func didTapImports() { + let settingsView = ImportStatusView(updatesController:updatesController, viewModel: ImportViewModel(), delegate: self) + + let hostingController = UIHostingController(rootView: settingsView) + let navigationController = UINavigationController(rootViewController: hostingController) + + self.closeMenu() + self.present(navigationController, animated: true) + } + public func didTapSettings() { let settingsView = PVSettingsView( conflictsController: updatesController, @@ -72,8 +85,13 @@ extension PVRootViewController: PVMenuDelegate { public func didTapAddGames() { self.closeMenu() - #if os(iOS) || os(tvOS) + + self.showImportOptionsAlert() + } + + public func showImportOptionsAlert() { +#if os(iOS) || os(tvOS) /// from PVGameLibraryViewController#getMoreROMs let actionSheet = UIAlertController(title: "Select Import Source", message: nil, preferredStyle: .actionSheet) #if !os(tvOS) @@ -214,4 +232,19 @@ extension PVRootViewController: UIDocumentPickerDelegate { ILOG("Document picker was cancelled") } } + +extension PVRootViewController: ImportStatusDelegate { + public func dismissAction() { + self.dismiss(animated: true) + } + + public func addImportsAction() { +// self.showImportOptionsAlert() + + let importItem = ImportItem(url: URL(fileURLWithPath: "game.rom"), + fileType: .game, + system: "NES") + gameImporter.addImport(importItem) + } +} #endif diff --git a/PVUI/Sources/PVSwiftUI/SideMenu/SideMenuView.swift b/PVUI/Sources/PVSwiftUI/SideMenu/SideMenuView.swift index 08b2194935..c5c0faae94 100644 --- a/PVUI/Sources/PVSwiftUI/SideMenu/SideMenuView.swift +++ b/PVUI/Sources/PVSwiftUI/SideMenu/SideMenuView.swift @@ -109,6 +109,10 @@ SideMenuView: SwiftUI.View { StatusBarProtectionWrapper { ScrollView { LazyVStack(alignment: .leading, spacing: 0) { + MenuItemView(imageName: "prov_settings_gear", rowTitle: "Imports") { + delegate.didTapImports() + } + Divider() MenuItemView(imageName: "prov_settings_gear", rowTitle: "Settings") { delegate.didTapSettings() } diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift index 8352a439f7..fb078e45da 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift @@ -27,7 +27,8 @@ public final class PVGameLibraryUpdatesController: ObservableObject { @Published public var conflicts: [ConflictsController.ConflictItem] = [] - private let gameImporter: GameImporter + public let gameImporter: GameImporter + private let directoryWatcher: DirectoryWatcher private let conflictsWatcher: ConflictsWatcher private let biosWatcher: BIOSWatcher From 78b1e7b862741b1a19ca968e71fe3c760fbc2ec0 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 3 Nov 2024 21:01:27 -0500 Subject: [PATCH 02/30] broke apart game importer files into smaller extensions in preparation to strange everything through the import queue --- .../GameImporterError.swift | 0 .../Importer/Models/ImportQueueItem.swift | 66 + .../ImporterArchiveType.swift | 0 .../ImporterFileType.swift | 0 .../MatchHashType.swift | 2 +- .../GameImporter => Models}/MatchType.swift | 2 +- .../GameImporter/GameImporter+Artwork.swift | 193 ++ .../GameImporter/GameImporter+Conflicts.swift | 167 ++ .../GameImporter/GameImporter+Files.swift | 304 +++ .../GameImporter/GameImporter+Importing.swift | 291 +++ .../GameImporter/GameImporter+Roms.swift | 145 ++ .../GameImporter/GameImporter+Systems.swift | 505 +++++ .../GameImporter/GameImporter+Utils.swift | 59 + .../Services/GameImporter/GameImporter.swift | 1741 +---------------- .../PVSwiftUI/Imports/ImportStatusView.swift | 19 +- .../PVSwiftUI/RootView/PVMenuDelegate.swift | 7 +- .../PVGameLibraryUpdatesController.swift | 8 +- 17 files changed, 1850 insertions(+), 1659 deletions(-) rename PVLibrary/Sources/PVLibrary/Importer/{Services/GameImporter => Models}/GameImporterError.swift (100%) create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift rename PVLibrary/Sources/PVLibrary/Importer/{Services/GameImporter => Models}/ImporterArchiveType.swift (100%) rename PVLibrary/Sources/PVLibrary/Importer/{Services/GameImporter => Models}/ImporterFileType.swift (100%) rename PVLibrary/Sources/PVLibrary/Importer/{Services/GameImporter => Models}/MatchHashType.swift (66%) rename PVLibrary/Sources/PVLibrary/Importer/{Services/GameImporter => Models}/MatchType.swift (84%) create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterError.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift similarity index 100% rename from PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterError.swift rename to PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift new file mode 100644 index 0000000000..68d1e11e92 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift @@ -0,0 +1,66 @@ +// +// ImportStatus.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +import SwiftUI + +// Enum to define the possible statuses of each import +public enum ImportStatus: String { + case queued + case processing + case success + case failure + case conflict // Indicates additional action needed by user after successful import + + public var description: String { + switch self { + case .queued: return "Queued" + case .processing: return "Processing" + case .success: return "Completed" + case .failure: return "Failed" + case .conflict: return "Conflict" + } + } + + public var color: Color { + switch self { + case .queued: return .gray + case .processing: return .blue + case .success: return .green + case .failure: return .red + case .conflict: return .yellow + } + } +} + +// Enum to define file types for each import +public enum FileType { + case bios, artwork, game, cdRom, unknown +} + +// Enum to track processing state +public enum ProcessingState { + case idle + case processing +} + +// ImportItem model to hold each file's metadata and progress +@Observable +public class ImportQueueItem: Identifiable, ObservableObject { + public let id = UUID() + public let url: URL + public var fileType: FileType + public var system: String // Can be set to the specific system type + + // Observable status for individual imports + public var status: ImportStatus = .queued + + public init(url: URL, fileType: FileType = .unknown, system: String = "") { + self.url = url + self.fileType = fileType + self.system = system + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ImporterArchiveType.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImporterArchiveType.swift similarity index 100% rename from PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ImporterArchiveType.swift rename to PVLibrary/Sources/PVLibrary/Importer/Models/ImporterArchiveType.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ImporterFileType.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImporterFileType.swift similarity index 100% rename from PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ImporterFileType.swift rename to PVLibrary/Sources/PVLibrary/Importer/Models/ImporterFileType.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/MatchHashType.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/MatchHashType.swift similarity index 66% rename from PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/MatchHashType.swift rename to PVLibrary/Sources/PVLibrary/Importer/Models/MatchHashType.swift index a2fc7005a3..255d5e274d 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/MatchHashType.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/MatchHashType.swift @@ -7,7 +7,7 @@ import Foundation -enum MatchHashType: String, Codable, Equatable, Hashable, Sendable { +package enum MatchHashType: String, Codable, Equatable, Hashable, Sendable { case MD5 case CRC case SHA1 diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/MatchType.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/MatchType.swift similarity index 84% rename from PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/MatchType.swift rename to PVLibrary/Sources/PVLibrary/Importer/Models/MatchType.swift index 7d4ac49417..23e95aff97 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/MatchType.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/MatchType.swift @@ -7,7 +7,7 @@ import Foundation -enum MatchType: Sendable { +package enum MatchType: Sendable { case byExtension case byHash(MatchHashType) case byFolder diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift new file mode 100644 index 0000000000..8eeaa0e9a4 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift @@ -0,0 +1,193 @@ +// +// func.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + + +public extension GameImporter { + /// Imports artwork from a given path + class func importArtwork(fromPath imageFullPath: URL) async -> PVGame? { + var isDirectory: ObjCBool = false + let fileExists = FileManager.default.fileExists(atPath: imageFullPath.path, isDirectory: &isDirectory) + if !fileExists || isDirectory.boolValue { + WLOG("File doesn't exist or is directory at \(imageFullPath)") + return nil + } + + var success = false + + defer { + if success { + do { + try FileManager.default.removeItem(at: imageFullPath) + } catch { + ELOG("Failed to delete image at path \(imageFullPath) \n \(error.localizedDescription)") + } + } + } + + let gameFilename: String = imageFullPath.deletingPathExtension().lastPathComponent + let gameExtension = imageFullPath.deletingPathExtension().pathExtension + let database = RomDatabase.sharedInstance + + if gameExtension.isEmpty { + ILOG("Trying to import artwork that didn't contain the extension of the system") + let games = database.all(PVGame.self, filter: NSPredicate(format: "romPath CONTAINS[c] %@", argumentArray: [gameFilename])) + + if games.count == 1, let game = games.first { + ILOG("File for image didn't have extension for system but we found a single match for image \(imageFullPath.lastPathComponent) to game \(game.title) on system \(game.systemIdentifier)") + guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { + return nil + } + + do { + try database.writeTransaction { + game.customArtworkURL = hash + } + success = true + ILOG("Set custom artwork of game \(game.title) from file \(imageFullPath.lastPathComponent)") + } catch { + ELOG("Couldn't update game \(game.title) with new artwork URL \(hash)") + } + + return game + } else { + VLOG("Database search returned \(games.count) results") + } + } + + guard let systems: [PVSystem] = PVEmulatorConfiguration.systemsFromCache(forFileExtension: gameExtension), !systems.isEmpty else { + ELOG("No system for extension \(gameExtension)") + return nil + } + + let cdBasedSystems = PVEmulatorConfiguration.cdBasedSystems + let couldBelongToCDSystem = !Set(cdBasedSystems).isDisjoint(with: Set(systems)) + + if (couldBelongToCDSystem && PVEmulatorConfiguration.supportedCDFileExtensions.contains(gameExtension.lowercased())) || systems.count > 1 { + guard let existingGames = findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: gameFilename) else { + ELOG("System for extension \(gameExtension) is a CD system and {\(gameExtension)} not the right matching file type of cue or m3u") + return nil + } + if existingGames.count == 1, let onlyMatch = existingGames.first { + ILOG("We found a hit for artwork that could have been belonging to multiple games and only found one file that matched by systemid/filename. The winner is \(onlyMatch.title) for \(onlyMatch.systemIdentifier)") + + guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { + ELOG("Couldn't move image, fail to set custom artwork") + return nil + } + + do { + try database.writeTransaction { + onlyMatch.customArtworkURL = hash + } + success = true + } catch { + ELOG("Couldn't update game \(onlyMatch.title) with new artwork URL") + } + return onlyMatch + } else { + ELOG("We got to the unlikely scenario where an extension is possibly a CD binary file, or belongs to a system, and had multiple games that matched the filename under more than one core.") + return nil + } + } + + guard let system = systems.first else { + ELOG("systems empty") + return nil + } + + var gamePartialPath: String = URL(fileURLWithPath: system.identifier, isDirectory: true).appendingPathComponent(gameFilename).deletingPathExtension().path + if gamePartialPath.first == "/" { + gamePartialPath.removeFirst() + } + + if gamePartialPath.isEmpty { + ELOG("Game path was empty") + return nil + } + + var games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), value: gamePartialPath) + if games.isEmpty { + games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), beginsWith: gamePartialPath) + } + + guard !games.isEmpty else { + ELOG("Couldn't find game for path \(gamePartialPath)") + return nil + } + + if games.count > 1 { + WLOG("There were multiple matches for \(gamePartialPath)! #\(games.count). Going with first for now until we make better code to prompt user.") + } + + let game = games.first! + + guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { + ELOG("scaleAndMoveImageToCache failed") + return nil + } + + do { + try database.writeTransaction { + game.customArtworkURL = hash + } + success = true + } catch { + ELOG("Couldn't update game with new artwork URL") + } + + return game + } + + /// Scales and moves an image to the cache + fileprivate class func scaleAndMoveImageToCache(imageFullPath: URL) -> String? { + let coverArtFullData: Data + do { + coverArtFullData = try Data(contentsOf: imageFullPath, options: []) + } catch { + ELOG("Couldn't read data from image file \(imageFullPath.path)\n\(error.localizedDescription)") + return nil + } + +#if canImport(UIKit) + guard let coverArtFullImage = UIImage(data: coverArtFullData) else { + ELOG("Failed to create Image from data") + return nil + } + guard let coverArtScaledImage = coverArtFullImage.scaledImage(withMaxResolution: Int(PVThumbnailMaxResolution)) else { + ELOG("Failed to create scale image") + return nil + } +#else + guard let coverArtFullImage = NSImage(data: coverArtFullData) else { + ELOG("Failed to create Image from data") + return nil + } + let coverArtScaledImage = coverArtFullImage +#endif + +#if canImport(UIKit) + guard let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) else { + ELOG("Failed to create data representation of scaled image") + return nil + } +#else + let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) +#endif + + let hash: String = (coverArtScaledData as NSData).md5 + + do { + let destinationURL = try PVMediaCache.writeData(toDisk: coverArtScaledData, withKey: hash) + VLOG("Scaled and moved image from \(imageFullPath.path) to \(destinationURL.path)") + } catch { + ELOG("Failed to save artwork to cache: \(error.localizedDescription)") + return nil + } + + return hash + } +} \ No newline at end of file diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift new file mode 100644 index 0000000000..c707eabc27 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift @@ -0,0 +1,167 @@ +// +// GameImporter+Conflicts.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +import Foundation +import PVSupport +import RealmSwift +import PVCoreLoader +import AsyncAlgorithms +import PVPlists +import PVLookup +import Systems +import PVMediaCache +import PVFileSystem +import PVLogging +import PVPrimitives +import PVRealm +import Perception +import SwiftUI + +extension GameImporter { + /// Resolves conflicts with given solutions + public func resolveConflicts(withSolutions solutions: [URL: System]) async { + let importOperation = BlockOperation() + + await solutions.asyncForEach { filePath, system in + let subfolder = system.romsDirectory + + if !FileManager.default.fileExists(atPath: subfolder.path, isDirectory: nil) { + ILOG("Path <\(subfolder.path)> doesn't exist. Creating.") + do { + try FileManager.default.createDirectory(at: subfolder, withIntermediateDirectories: true, attributes: nil) + } catch { + ELOG("Error making conflicts dir <\(subfolder.path)>") + assertionFailure("Error making conflicts dir <\(subfolder.path)>") + } + } + + let sourceFilename: String = filePath.lastPathComponent + let sourcePath: URL = filePath + let destinationPath: URL = subfolder.appendingPathComponent(sourceFilename, isDirectory: false) + + do { + try moveAndOverWrite(sourcePath: sourcePath, destinationPath: destinationPath) + } catch { + ELOG("\(error)") + } + + let relatedFileName: String = sourcePath.deletingPathExtension().lastPathComponent + + let conflictsDirContents = try? FileManager.default.contentsOfDirectory(at: conflictPath, includingPropertiesForKeys: nil, options: []) + conflictsDirContents?.forEach { file in + var fileWithoutExtension: String = file.deletingPathExtension().lastPathComponent + fileWithoutExtension = PVEmulatorConfiguration.stripDiscNames(fromFilename: fileWithoutExtension) + let relatedFileName = PVEmulatorConfiguration.stripDiscNames(fromFilename: relatedFileName) + + if fileWithoutExtension == relatedFileName { + let isCueSheet = destinationPath.pathExtension == "cue" + + if isCueSheet { + let cueSheetPath = destinationPath + if var cuesheetContents = try? String(contentsOf: cueSheetPath, encoding: .utf8) { + let range = (cuesheetContents as NSString).range(of: file.lastPathComponent, options: .caseInsensitive) + + if range.location != NSNotFound { + if let subRange = Range(range, in: cuesheetContents) { + cuesheetContents.replaceSubrange(subRange, with: file.lastPathComponent) + } + + do { + try cuesheetContents.write(to: cueSheetPath, atomically: true, encoding: .utf8) + } catch { + ELOG("Unable to rewrite cuesheet \(destinationPath.path) because \(error.localizedDescription)") + } + } else { + DLOG("Range of string <\(file)> not found in file <\(cueSheetPath.lastPathComponent)>") + } + } + } + + do { + let newDestinationPath = subfolder.appendingPathComponent(file.lastPathComponent, isDirectory: false) + try moveAndOverWrite(sourcePath: file, destinationPath: newDestinationPath) + NSLog("Moving \(file.lastPathComponent) to \(newDestinationPath)") + } catch { + ELOG("Unable to move related file from \(filePath.path) to \(subfolder.path) because: \(error.localizedDescription)") + } + } + } + + importOperation.addExecutionBlock { + Task { + ILOG("Import Files at \(destinationPath)") + if let system = RomDatabase.systemCache[system.identifier] { + RomDatabase.addFileSystemROMCache(system) + } + await self.getRomInfoForFiles(atPaths: [destinationPath], userChosenSystem: system) + } + } + } + + let completionOperation = BlockOperation { + if self.completionHandler != nil { + DispatchQueue.main.async(execute: { () -> Void in + self.completionHandler?(false) + }) + } + } + + completionOperation.addDependency(importOperation) + serialImportQueue.addOperation(importOperation) + serialImportQueue.addOperation(completionOperation) + } + + /// Handles a system conflict + internal func handleSystemConflict(path: URL, systems: [PVSystem]) async throws { + let candidate = ImportCandidateFile(filePath: path) + DLOG("Handling system conflict for path: \(path.lastPathComponent)") + DLOG("Possible systems: \(systems.map { $0.name }.joined(separator: ", "))") + + // Try to determine system using all available methods + if let system = try? await determineSystem(for: candidate) { + if systems.contains(system) { + DLOG("Found matching system: \(system.name)") + try importGame(path: path, system: system) + return + } else { + DLOG("Determined system \(system.name) not in possible systems list") + } + } else { + DLOG("Could not determine system automatically") + } + + // Fall back to multiple system handling + DLOG("Falling back to multiple system handling") + try handleMultipleSystemMatch(path: path, systems: systems) + } + + /// Handles a multiple system match + internal func handleMultipleSystemMatch(path: URL, systems: [PVSystem]) throws { + let filename = path.lastPathComponent + guard let existingGames = GameImporter.findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: filename) else { + self.encounteredConflicts = true + try moveToConflictsDirectory(path: path) + return + } + + if existingGames.count == 1 { + try importGame(path: path, system: systems.first!) + } else { + self.encounteredConflicts = true + try moveToConflictsDirectory(path: path) + let matchedSystems = systems.map { $0.identifier }.joined(separator: ", ") + let matchedGames = existingGames.map { $0.romPath }.joined(separator: ", ") + WLOG("Scanned game matched with multiple systems {\(matchedSystems)} and multiple existing games \(matchedGames) so we moved \(filename) to conflicts dir. You figure it out!") + } + } + + /// Moves a file to the conflicts directory + internal func moveToConflictsDirectory(path: URL) throws { + let destination = conflictPath.appendingPathComponent(path.lastPathComponent) + try moveAndOverWrite(sourcePath: path, destinationPath: destination) + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift new file mode 100644 index 0000000000..e38141e303 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift @@ -0,0 +1,304 @@ +// +// GameImporter+Files.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +import Foundation +import PVSupport +import RealmSwift + +extension GameImporter { + /// Moves a CD-ROM to the appropriate subfolder + internal func moveCDROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> [URL]? { + guard isCDROM(candidate.filePath) else { + return nil + } + + let fileManager = FileManager.default + let fileName = candidate.filePath.lastPathComponent + + guard let system = try? await determineSystem(for: candidate) else { + throw GameImporterError.unsupportedSystem + } + + let destinationFolder = system.romsDirectory + let destinationPath = destinationFolder.appendingPathComponent(fileName) + + do { + try fileManager.createDirectory(at: destinationFolder, withIntermediateDirectories: true, attributes: nil) + try fileManager.moveItem(at: candidate.filePath, to: destinationPath) + let relatedFiles = try await moveRelatedFiles(for: candidate, to: destinationFolder) + return [destinationPath] + relatedFiles + } catch { + throw GameImporterError.failedToMoveCDROM(error) + } + } + + /// Moves a ROM to the appropriate subfolder + internal func moveROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> URL? { + guard !isCDROM(candidate.filePath) else { + return nil + } + + let fileManager = FileManager.default + let fileName = candidate.filePath.lastPathComponent + + // Check first if known BIOS + if let system = try await handleBIOSFile(candidate) { + DLOG("Moving BIOS file to system: \(system.name)") + let destinationFolder = system.biosDirectory + let destinationPath = destinationFolder.appendingPathComponent(fileName) + return try await moveROMFile(candidate, to: destinationPath) + } + + // Check for M3U + if let system = try await handleM3UFile(candidate) { + DLOG("Moving M3U and referenced files to system: \(system.name)") + // Move M3U and all referenced files to system directory + let destinationDir = system.romsDirectory + return try await moveM3UAndReferencedFiles(candidate, to: destinationDir) + } + + // CD-ROM handling + if let system = try await handleCDROMFile(candidate) { + DLOG("Moving CD-ROM files to system: \(system.name)") + let destinationDir = system.romsDirectory + return try await moveCDROMFiles(candidate, to: destinationDir) + } + + // Regular ROM handling + let (system, hasConflict) = try await handleRegularROM(candidate) + let destinationDir = hasConflict ? self.conflictPath : system.romsDirectory + + DLOG("Moving ROM to \(hasConflict ? "conflicts" : "system") directory: \(system.name)") + return try await moveROMFile(candidate, to: destinationDir) + } + + private func handleBIOSFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { + guard let md5 = candidate.md5?.uppercased() else { + return nil + } + + // Get all BIOS entries that match this MD5 + let matchingBIOSEntries = PVEmulatorConfiguration.biosEntries.filter { biosEntry in + let frozenBiosEntry = biosEntry.isFrozen ? biosEntry : biosEntry.freeze() + return frozenBiosEntry.expectedMD5.uppercased() == md5 + } + + if !matchingBIOSEntries.isEmpty { + // Get the first matching system + if let firstBIOSEntry = matchingBIOSEntries.first { + let frozenBiosEntry = firstBIOSEntry.isFrozen ? firstBIOSEntry : firstBIOSEntry.freeze() + + // Move file to BIOS directory + let destinationURL = frozenBiosEntry.expectedPath + try await moveROMFile(candidate, to: destinationURL) + + // Update BIOS entry in Realm + try await MainActor.run { + let realm = try Realm() + try realm.write { + if let thawedBios = frozenBiosEntry.thaw() { + let biosFile = PVFile(withURL: destinationURL) + thawedBios.file = biosFile + } + } + } + + return frozenBiosEntry.system + } + } + + return nil + } + + private func handleCDROMFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { + let `extension` = candidate.filePath.pathExtension.lowercased() + guard PVEmulatorConfiguration.supportedCDFileExtensions.contains(`extension`) else { + return nil + } + + DLOG("Handling CD-ROM file: \(candidate.filePath.lastPathComponent)") + + // First try MD5 matching + if let system = try? await determineSystem(for: candidate) { + DLOG("Found system match for CD-ROM by MD5: \(system.name)") + return system + } + + // If cue file, try to match its bin file + if `extension` == "cue" { + if let binFile = try findAssociatedBinFile(for: candidate) { + DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") + let binCandidate = ImportCandidateFile(filePath: binFile) + if let system = try? await determineSystem(for: binCandidate) { + DLOG("Found system match from associated bin file: \(system.name)") + return system + } + } + } + + // Try exact filename match + if let system = await matchSystemByFileName(candidate.filePath.lastPathComponent) { + DLOG("Found system match by filename: \(system.name)") + return system + } + + DLOG("No system match found for CD-ROM file") + return nil + } + + /// Move a `ImportCandidateFile` to a destination, creating the destination directory if needed + private func moveROMFile(_ romFile: ImportCandidateFile, to destination: URL) async throws -> URL { + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let destPath = destination.appendingPathComponent(romFile.filePath.lastPathComponent) + try FileManager.default.moveItem(at: romFile.filePath, to: destPath) + DLOG("Moved ROM file to: \(destPath.path)") + return destPath + } + + private func findAssociatedBinFile(for cueFile: ImportCandidateFile) throws -> URL? { + let cueContents = try String(contentsOf: cueFile.filePath, encoding: .utf8) + let lines = cueContents.components(separatedBy: .newlines) + + // Look for FILE "something.bin" BINARY line + for line in lines { + let components = line.trimmingCharacters(in: .whitespaces) + .components(separatedBy: "\"") + guard components.count >= 2, + line.lowercased().contains("file") && line.lowercased().contains("binary") else { + continue + } + + let binFileName = components[1] + let binPath = cueFile.filePath.deletingLastPathComponent().appendingPathComponent(binFileName) + + if FileManager.default.fileExists(atPath: binPath.path) { + return binPath + } + } + + return nil + } + + private func moveM3UAndReferencedFiles(_ m3uFile: ImportCandidateFile, to destination: URL) async throws -> URL { + let contents = try String(contentsOf: m3uFile.filePath, encoding: .utf8) + let files = contents.components(separatedBy: .newlines) + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && !$0.hasPrefix("#") } + + // Create destination directory if needed + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + + // Move all referenced files + for file in files { + let sourcePath = m3uFile.filePath.deletingLastPathComponent().appendingPathComponent(file) + let destPath = destination.appendingPathComponent(file) + + if FileManager.default.fileExists(atPath: sourcePath.path) { + try FileManager.default.moveItem(at: sourcePath, to: destPath) + DLOG("Moved M3U referenced file: \(file)") + } + } + + // Move M3U file itself + let m3uDestPath = destination.appendingPathComponent(m3uFile.filePath.lastPathComponent) + try FileManager.default.moveItem(at: m3uFile.filePath, to: m3uDestPath) + DLOG("Moved M3U file to: \(m3uDestPath.path)") + + return m3uDestPath + } + + private func moveCDROMFiles(_ cdFile: ImportCandidateFile, to destination: URL) async throws -> URL { + let fileManager = FileManager.default + try fileManager.createDirectory(at: destination, withIntermediateDirectories: true) + + let `extension` = cdFile.filePath.pathExtension.lowercased() + let destPath = destination.appendingPathComponent(cdFile.filePath.lastPathComponent) + + // If it's a cue file, move both cue and bin + if `extension` == "cue" { + if let binPath = try findAssociatedBinFile(for: cdFile) { + let binDestPath = destination.appendingPathComponent(binPath.lastPathComponent) + try fileManager.moveItem(at: binPath, to: binDestPath) + DLOG("Moved bin file to: \(binDestPath.path)") + } + } + + // Move the main CD-ROM file + try fileManager.moveItem(at: cdFile.filePath, to: destPath) + DLOG("Moved CD-ROM file to: \(destPath.path)") + + return destPath + } + + /// Moves related files for a given candidate + private func moveRelatedFiles(for candidate: ImportCandidateFile, to destinationFolder: URL) async throws -> [URL] { + let fileManager = FileManager.default + let fileName = candidate.filePath.deletingPathExtension().lastPathComponent + let sourceFolder = candidate.filePath.deletingLastPathComponent() + + let relatedFiles = try fileManager.contentsOfDirectory(at: sourceFolder, includingPropertiesForKeys: nil) + .filter { $0.deletingPathExtension().lastPathComponent == fileName && $0 != candidate.filePath } + + return try await withThrowingTaskGroup(of: URL.self) { group in + for file in relatedFiles { + group.addTask { + let destination = destinationFolder.appendingPathComponent(file.lastPathComponent) + try fileManager.moveItem(at: file, to: destination) + return destination + } + } + + var movedFiles: [URL] = [] + for try await movedFile in group { + movedFiles.append(movedFile) + } + return movedFiles + } + } + + /// Moves a file and overwrites if it already exists at the destination + public func moveAndOverWrite(sourcePath: URL, destinationPath: URL) throws { + let fileManager = FileManager.default + + // If file exists at destination, remove it first + if fileManager.fileExists(atPath: destinationPath.path) { + try fileManager.removeItem(at: destinationPath) + } + + // Now move the file + try fileManager.moveItem(at: sourcePath, to: destinationPath) + } + + /// BIOS entry matching + private func biosEntryMatching(candidateFile: ImportCandidateFile) -> [PVBIOS]? { + let fileName = candidateFile.filePath.lastPathComponent + var matchingBioses = Set() + + DLOG("Checking if file is BIOS: \(fileName)") + + // First try to match by filename + if let biosEntry = PVEmulatorConfiguration.biosEntry(forFilename: fileName) { + DLOG("Found BIOS match by filename: \(biosEntry.expectedFilename)") + matchingBioses.insert(biosEntry) + } + + // Then try to match by MD5 + if let md5 = candidateFile.md5?.uppercased(), + let md5Entry = PVEmulatorConfiguration.biosEntry(forMD5: md5) { + DLOG("Found BIOS match by MD5: \(md5Entry.expectedFilename)") + matchingBioses.insert(md5Entry) + } + + if !matchingBioses.isEmpty { + let matches = Array(matchingBioses) + DLOG("Found \(matches.count) BIOS matches") + return matches + } + + return nil + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift new file mode 100644 index 0000000000..4264904f41 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift @@ -0,0 +1,291 @@ +// +// GameImporter+Importing.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +import Foundation +import PVSupport +import RealmSwift +import PVCoreLoader +import AsyncAlgorithms +import PVPlists +import PVLookup +import Systems +import PVMediaCache +import PVFileSystem +import PVLogging +import PVPrimitives +import PVRealm +import Perception +import SwiftUI + +extension GameImporter { + /// Imports files from given paths + /// //goal is to make this private + public func importFiles(atPaths paths: [URL]) async throws -> [URL] { + let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) + var importedFiles: [URL] = [] + + for path in sortedPaths { + do { + if let importedFile = try await importSingleFile(at: path) { + importedFiles.append(importedFile) + } + } catch { + ELOG("Failed to import file at \(path.path): \(error.localizedDescription)") + } + } + + return importedFiles + } + + /// Imports a single file from the given path + internal func importSingleFile(at path: URL) async throws -> URL? { + guard FileManager.default.fileExists(atPath: path.path) else { + WLOG("File doesn't exist at \(path.path)") + return nil + } + + if isCDROM(path) { + return try await handleCDROM(at: path) + } else if isArtwork(path) { + return try await handleArtwork(at: path) + } else { + return try await handleROM(at: path) + } + } + + /// Handles importing a CD-ROM + internal func handleCDROM(at path: URL) async throws -> URL? { + let movedToPaths = try await moveCDROM(toAppropriateSubfolder: ImportCandidateFile(filePath: path)) + if let movedToPaths = movedToPaths { + let pathsString = movedToPaths.map { $0.path }.joined(separator: ", ") + VLOG("Found a CD. Moved files to the following paths \(pathsString)") + } + return nil + } + + /// Handles importing artwork + internal func handleArtwork(at path: URL) async throws -> URL? { + if let game = await GameImporter.importArtwork(fromPath: path) { + ILOG("Found artwork \(path.lastPathComponent) for game <\(game.title)>") + } + return nil + } + + /// Handles importing a ROM + internal func handleROM(at path: URL) async throws -> URL? { + let candidate = ImportCandidateFile(filePath: path) + return try await moveROM(toAppropriateSubfolder: candidate) + } + + /// Starts an import for the given paths + internal func startImport(forPaths paths: [URL]) async { + // Pre-sort + let paths = PVEmulatorConfiguration.sortImportURLs(urls: paths) + let scanOperation = BlockOperation { + Task { + do { + let newPaths = try await self.importFiles(atPaths: paths) + await self.getRomInfoForFiles(atPaths: newPaths, userChosenSystem: nil) + } catch { + ELOG("\(error)") + } + } + } + + let completionOperation = BlockOperation { + if self.completionHandler != nil { + DispatchQueue.main.sync(execute: { () -> Void in + self.completionHandler?(self.encounteredConflicts) + }) + } + } + + completionOperation.addDependency(scanOperation) + serialImportQueue.addOperation(scanOperation) + serialImportQueue.addOperation(completionOperation) + } + + /// Handles the import of a path + internal func _handlePath(path: URL, userChosenSystem chosenSystem: System?) async throws { + // Skip hidden files and directories + if path.lastPathComponent.hasPrefix(".") { + VLOG("Skipping hidden file or directory: \(path.lastPathComponent)") + return + } + + let isDirectory = path.hasDirectoryPath + let filename = path.lastPathComponent + let fileExtensionLower = path.pathExtension.lowercased() + + // Handle directories + //TODO: strangle this back to the importer queue somehow... + if isDirectory { + try await handleDirectory(path: path, chosenSystem: chosenSystem) + return + } + + // Handle files + let systems = try determineSystems(for: path, chosenSystem: chosenSystem) + + // Handle conflicts + if systems.count > 1 { + try await handleSystemConflict(path: path, systems: systems) + return + } + + //this is the case where there was no matching system - should this even happne? + guard let system = systems.first else { + ELOG("No system matched extension {\(fileExtensionLower)}") + try moveToConflictsDirectory(path: path) + return + } + + try importGame(path: path, system: system) + } + + /// Handles a directory + /// //TODO: strangle this back to the importer queue + internal func handleDirectory(path: URL, chosenSystem: System?) async throws { + guard chosenSystem == nil else { return } + + do { + let subContents = try FileManager.default.contentsOfDirectory(at: path, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) + if subContents.isEmpty { + try await FileManager.default.removeItem(at: path) + ILOG("Deleted empty import folder \(path.path)") + } else { + ILOG("Found non-empty folder in imports dir. Will iterate subcontents for import") + for subFile in subContents { + try await self._handlePath(path: subFile, userChosenSystem: nil) + } + } + } catch { + ELOG("Error handling directory: \(error)") + throw error + } + } + + internal func handleM3UFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { + guard candidate.filePath.pathExtension.lowercased() == "m3u" else { + return nil + } + + DLOG("Handling M3U file: \(candidate.filePath.lastPathComponent)") + + // First try to match the M3U file itself by MD5 + if let system = try? await determineSystem(for: candidate) { + DLOG("Found system match for M3U by MD5: \(system.name)") + return system + } + + // Read M3U contents + let contents = try String(contentsOf: candidate.filePath, encoding: .utf8) + let files = contents.components(separatedBy: .newlines) + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && !$0.hasPrefix("#") } + + DLOG("Found \(files.count) entries in M3U") + + // Try to match first valid file in M3U + for file in files { + let filePath = candidate.filePath.deletingLastPathComponent().appendingPathComponent(file) + guard FileManager.default.fileExists(atPath: filePath.path) else { continue } + + let candidateFile = ImportCandidateFile(filePath: filePath) + if let system = try? await determineSystem(for: candidateFile) { + DLOG("Found system match from M3U entry: \(file) -> \(system.name)") + return system + } + } + + DLOG("No system match found for M3U or its contents") + return nil + } + + internal func handleRegularROM(_ candidate: ImportCandidateFile) async throws -> (PVSystem, Bool) { + DLOG("Handling regular ROM file: \(candidate.filePath.lastPathComponent)") + + // 1. Try MD5 match first + if let md5 = candidate.md5?.uppercased() { + if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), + !results.isEmpty { + let matchingSystems = results.compactMap { result -> PVSystem? in + guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } + return PVEmulatorConfiguration.system(forIdentifier: sysID) + } + + if matchingSystems.count == 1 { + DLOG("Found single system match by MD5: \(matchingSystems[0].name)") + return (matchingSystems[0], false) + } else if matchingSystems.count > 1 { + DLOG("Found multiple system matches by MD5, moving to conflicts") + return (matchingSystems[0], true) // Return first with conflict flag + } + } + } + + let fileName = candidate.filePath.lastPathComponent + let fileExtension = candidate.filePath.pathExtension.lowercased() + let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: fileExtension) ?? [] + + // 2. Try exact filename match + if let system = await matchSystemByFileName(fileName) { + DLOG("Found system match by exact filename: \(system.name)") + return (system, false) + } + + // 3. Try extension match + if possibleSystems.count == 1 { + DLOG("Single system match by extension: \(possibleSystems[0].name)") + return (possibleSystems[0], false) + } else if possibleSystems.count > 1 { + DLOG("Multiple systems match extension, trying partial name match") + + // 4. Try partial filename system identifier match + if let system = matchSystemByPartialName(fileName, possibleSystems: possibleSystems) { + DLOG("Found system match by partial name: \(system.name)") + return (system, false) + } + + DLOG("No definitive system match, moving to conflicts") + return (possibleSystems[0], true) + } + + throw GameImporterError.systemNotDetermined + } + + /// Finishes the update or import of a game + internal func finishUpdateOrImport(ofGame game: PVGame, path: URL) async throws { + // Only process if rom doensn't exist in DB + if await RomDatabase.gamesCache[game.romPath] != nil { + throw GameImporterError.romAlreadyExistsInDatabase + } + var modified = false + var game:PVGame = game + if game.requiresSync { + if importStartedHandler != nil { + let fullpath = PVEmulatorConfiguration.path(forGame: game) + Task { @MainActor in + self.importStartedHandler?(fullpath.path) + } + } + game = lookupInfo(for: game, overwrite: true) + modified = true + } + let wasModified = modified + if finishedImportHandler != nil { + let md5: String = game.md5Hash + // Task { @MainActor in + self.finishedImportHandler?(md5, wasModified) + // } + } + if game.originalArtworkFile == nil { + game = await getArtwork(forGame: game) + } + self.saveGame(game) + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift new file mode 100644 index 0000000000..134a0d13be --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift @@ -0,0 +1,145 @@ +// +// GameImporter+Roms.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +import Foundation +import PVSupport +import RealmSwift +import PVCoreLoader +import AsyncAlgorithms +import PVPlists +import PVLookup +import Systems +import PVMediaCache +import PVFileSystem +import PVLogging +import PVPrimitives +import PVRealm +import Perception +import SwiftUI + +public extension GameImporter { + + /// Retrieves ROM information for files at given paths + func getRomInfoForFiles(atPaths paths: [URL], userChosenSystem chosenSystem: System? = nil) async { + //TODO: split this off at the importer queue entry point so we can remove here + // If directory, map out sub directories if folder + let paths: [URL] = paths.compactMap { (url) -> [URL]? in + if url.hasDirectoryPath { + return try? FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) + } else { + return [url] + } + }.joined().map { $0 } + + let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) + await sortedPaths.asyncForEach { path in + do { + try await self._handlePath(path: path, userChosenSystem: chosenSystem) + } catch { + //TODO: what do i do here? I probably could just let this throw... + ELOG("\(error)") + } + } // for each + } + + internal func importGame(path: URL, system: PVSystem) throws { + DLOG("Attempting to import game: \(path.lastPathComponent) for system: \(system.name)") + let filename = path.lastPathComponent + let partialPath = (system.identifier as NSString).appendingPathComponent(filename) + let similarName = RomDatabase.altName(path, systemIdentifier: system.identifier) + + DLOG("Checking game cache for partialPath: \(partialPath) or similarName: \(similarName)") + let gamesCache = RomDatabase.gamesCache + + if let existingGame = gamesCache[partialPath] ?? gamesCache[similarName], + system.identifier == existingGame.systemIdentifier { + DLOG("Found existing game in cache, saving relative path") + saveRelativePath(existingGame, partialPath: partialPath, file: path) + } else { + DLOG("No existing game found, starting import to database") + Task.detached(priority: .utility) { + try await self.importToDatabaseROM(atPath: path, system: system, relatedFiles: nil) + } + } + } + + /// Imports a ROM to the database + internal func importToDatabaseROM(atPath path: URL, system: PVSystem, relatedFiles: [URL]?) async throws { + DLOG("Starting database ROM import for: \(path.lastPathComponent)") + let filename = path.lastPathComponent + let filenameSansExtension = path.deletingPathExtension().lastPathComponent + let title: String = PVEmulatorConfiguration.stripDiscNames(fromFilename: filenameSansExtension) + let destinationDir = (system.identifier as NSString) + let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) + + DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") + let file = PVFile(withURL: path) + let game = PVGame(withFile: file, system: system) + game.romPath = partialPath + game.title = title + game.requiresSync = true + var relatedPVFiles = [PVFile]() + let files = RomDatabase.getFileSystemROMCache(for: system).keys + let name = RomDatabase.altName(path, systemIdentifier: system.identifier) + + DLOG("Searching for related files with name: \(name)") + + await files.asyncForEach { url in + let relativeName = RomDatabase.altName(url, systemIdentifier: system.identifier) + DLOG("Checking file \(url.lastPathComponent) with relative name: \(relativeName)") + if relativeName == name { + DLOG("Found matching related file: \(url.lastPathComponent)") + relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) + } + } + + if let relatedFiles = relatedFiles { + DLOG("Processing \(relatedFiles.count) additional related files") + for url in relatedFiles { + DLOG("Adding related file: \(url.lastPathComponent)") + relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) + } + } + + guard let md5 = calculateMD5(forGame: game)?.uppercased() else { + ELOG("Couldn't calculate MD5 for game \(partialPath)") + throw GameImporterError.couldNotCalculateMD5 + } + DLOG("Calculated MD5: \(md5)") + + // Register import with coordinator + guard await importCoordinator.checkAndRegisterImport(md5: md5) else { + DLOG("Import already in progress for MD5: \(md5)") + throw GameImporterError.romAlreadyExistsInDatabase + } + DLOG("Registered import with coordinator for MD5: \(md5)") + + defer { + Task { + await importCoordinator.completeImport(md5: md5) + DLOG("Completed import coordination for MD5: \(md5)") + } + } + + game.relatedFiles.append(objectsIn: relatedPVFiles) + game.md5Hash = md5 + try await finishUpdateOrImport(ofGame: game, path: path) + } + + /// Saves a game to the database + func saveGame(_ game:PVGame) { + do { + let database = RomDatabase.sharedInstance + try database.writeTransaction { + database.realm.create(PVGame.self, value:game, update:.modified) + } + RomDatabase.addGamesCache(game) + } catch { + ELOG("Couldn't add new game \(error.localizedDescription)") + } + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift new file mode 100644 index 0000000000..2a28ad0bb2 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift @@ -0,0 +1,505 @@ +// +// GameImporter+Systems.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +import Foundation +import PVSupport +import RealmSwift +import PVCoreLoader +import AsyncAlgorithms +import PVPlists +import PVLookup +import Systems +import PVMediaCache +import PVFileSystem +import PVLogging +import PVPrimitives +import PVRealm +import Perception +import SwiftUI + +extension GameImporter { + + internal func matchSystemByPartialName(_ fileName: String, possibleSystems: [PVSystem]) -> PVSystem? { + let cleanedName = fileName.lowercased() + + for system in possibleSystems { + let patterns = filenamePatterns(forSystem: system) + + for pattern in patterns { + if (try? NSRegularExpression(pattern: pattern, options: .caseInsensitive))? + .firstMatch(in: cleanedName, options: [], range: NSRange(cleanedName.startIndex..., in: cleanedName)) != nil { + DLOG("Found system match by pattern '\(pattern)' for system: \(system.name)") + return system + } + } + } + + return nil + } + + /// Matches a system based on the file name + internal func matchSystemByFileName(_ fileName: String) async -> PVSystem? { + let systems = PVEmulatorConfiguration.systems + let lowercasedFileName = fileName.lowercased() + let fileExtension = (fileName as NSString).pathExtension.lowercased() + + // First, try to match based on file extension + if let systemsForExtension = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + if systemsForExtension.count == 1 { + return systemsForExtension[0] + } else if systemsForExtension.count > 1 { + // If multiple systems match the extension, try to narrow it down + for system in systemsForExtension { + if await doesFileNameMatch(lowercasedFileName, forSystem: system) { + return system + } + } + } + } + + // If extension matching fails, try other methods + for system in systems { + if await doesFileNameMatch(lowercasedFileName, forSystem: system) { + return system + } + } + + // If no match found, try querying the OpenVGDB + do { + if let results = try await openVGDB.searchDatabase(usingFilename: fileName), + let firstResult = results.first, + let systemID = firstResult["systemID"] as? Int, + let system = PVEmulatorConfiguration.system(forDatabaseID: systemID) { + return system + } + } catch { + ELOG("Error querying OpenVGDB for filename: \(error.localizedDescription)") + } + + return nil + } + + /// Checks if a file name matches a given system + private func doesFileNameMatch(_ lowercasedFileName: String, forSystem system: PVSystem) async -> Bool { + // Check if the filename contains the system's name or abbreviation + if lowercasedFileName.contains(system.name.lowercased()) || + lowercasedFileName.contains(system.shortName.lowercased()) { + return true + } + + // Check against known filename patterns for the system + let patterns = filenamePatterns(forSystem: system) + for pattern in patterns { + if lowercasedFileName.range(of: pattern, options: .regularExpression) != nil { + return true + } + } + + // Check against a list of known game titles for the system + if await isKnownGameTitle(lowercasedFileName, forSystem: system) { + return true + } + + return false + } + + /// Checks if a file name matches a known game title for a given system + private func isKnownGameTitle(_ lowercasedFileName: String, forSystem system: PVSystem) async -> Bool { + do { + // Remove file extension and common parenthetical information + let cleanedFileName = cleanFileName(lowercasedFileName) + + // Search the database using the cleaned filename + if let results = try await openVGDB.searchDatabase(usingFilename: cleanedFileName, systemID: system.openvgDatabaseID) { + // Check if we have any results + if !results.isEmpty { + // Optionally, you can add more strict matching here + for result in results { + if let gameTitle = result["gameTitle"] as? String, + cleanFileName(gameTitle.lowercased()) == cleanedFileName { + return true + } + } + } + } + } catch { + ELOG("Error searching OpenVGDB for known game title: \(error.localizedDescription)") + } + return false + } + + /// Cleans a file name + private func cleanFileName(_ fileName: String) -> String { + var cleaned = fileName.lowercased() + + // Remove file extension + if let dotIndex = cleaned.lastIndex(of: ".") { + cleaned = String(cleaned[.. [String] { + let systemName = system.name.lowercased() + let shortName = system.shortName.lowercased() + + var patterns: [String] = [] + + // Add pattern for full system name + patterns.append("\\b\(systemName)\\b") + + // Add pattern for short name + patterns.append("\\b\(shortName)\\b") + // Add some common variations and abbreviations + switch system.identifier { + case "com.provenance.nes": + patterns.append("\\b(nes|nintendo)\\b") + case "com.provenance.snes": + patterns.append("\\b(snes|super\\s*nintendo)\\b") + case "com.provenance.genesis": + patterns.append("\\b(genesis|mega\\s*drive|md)\\b") + case "com.provenance.gba": + patterns.append("\\b(gba|game\\s*boy\\s*advance)\\b") + case "com.provenance.n64": + patterns.append("\\b(n64|nintendo\\s*64)\\b") + case "com.provenance.psx": + patterns.append("\\b(psx|playstation|ps1)\\b") + case "com.provenance.ps2": + patterns.append("\\b(ps2|playstation\\s*2)\\b") + case "com.provenance.gb": + patterns.append("\\b(gb|game\\s*boy)\\b") + case "com.provenance.3DO": + patterns.append("\\b(3do|panasonic\\s*3do)\\b") + case "com.provenance.3ds": + patterns.append("\\b(3ds|nintendo\\s*3ds)\\b") + case "com.provenance.2600": + patterns.append("\\b(2600|atari\\s*2600|vcs)\\b") + case "com.provenance.5200": + patterns.append("\\b(5200|atari\\s*5200)\\b") + case "com.provenance.7800": + patterns.append("\\b(7800|atari\\s*7800)\\b") + case "com.provenance.jaguar": + patterns.append("\\b(jaguar|atari\\s*jaguar)\\b") + case "com.provenance.colecovision": + patterns.append("\\b(coleco|colecovision)\\b") + case "com.provenance.dreamcast": + patterns.append("\\b(dc|dreamcast|sega\\s*dreamcast)\\b") + case "com.provenance.ds": + patterns.append("\\b(nds|nintendo\\s*ds)\\b") + case "com.provenance.gamegear": + patterns.append("\\b(gg|game\\s*gear|sega\\s*game\\s*gear)\\b") + case "com.provenance.gbc": + patterns.append("\\b(gbc|game\\s*boy\\s*color)\\b") + case "com.provenance.lynx": + patterns.append("\\b(lynx|atari\\s*lynx)\\b") + case "com.provenance.mastersystem": + patterns.append("\\b(sms|master\\s*system|sega\\s*master\\s*system)\\b") + case "com.provenance.neogeo": + patterns.append("\\b(neo\\s*geo|neogeo|neo-geo)\\b") + case "com.provenance.ngp": + patterns.append("\\b(ngp|neo\\s*geo\\s*pocket)\\b") + case "com.provenance.ngpc": + patterns.append("\\b(ngpc|neo\\s*geo\\s*pocket\\s*color)\\b") + case "com.provenance.psp": + patterns.append("\\b(psp|playstation\\s*portable)\\b") + case "com.provenance.saturn": + patterns.append("\\b(saturn|sega\\s*saturn)\\b") + case "com.provenance.32X": + patterns.append("\\b(32x|sega\\s*32x)\\b") + case "com.provenance.segacd": + patterns.append("\\b(scd|sega\\s*cd|mega\\s*cd)\\b") + case "com.provenance.sg1000": + patterns.append("\\b(sg1000|sg-1000|sega\\s*1000)\\b") + case "com.provenance.vb": + patterns.append("\\b(vb|virtual\\s*boy)\\b") + case "com.provenance.ws": + patterns.append("\\b(ws|wonderswan)\\b") + case "com.provenance.wsc": + patterns.append("\\b(wsc|wonderswan\\s*color)\\b") + default: + // For systems without specific patterns, we'll just use the general ones created above + break + } + + return patterns + } + + /// Determines the system for a given candidate file + private func determineSystemFromContent(for candidate: ImportCandidateFile, possibleSystems: [PVSystem]) throws -> PVSystem { + // Implement logic to determine system based on file content or metadata + // This could involve checking file headers, parsing content, or using a database of known games + + let fileName = candidate.filePath.deletingPathExtension().lastPathComponent + + for system in possibleSystems { + do { + if let results = try openVGDB.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), + !results.isEmpty { + ILOG("System determined by filename match in OpenVGDB: \(system.name)") + return system + } + } catch { + ELOG("Error searching OpenVGDB for system \(system.name): \(error.localizedDescription)") + } + } + + // If we couldn't determine the system, try a more detailed search + if let fileMD5 = candidate.md5?.uppercased(), !fileMD5.isEmpty { + do { + if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: fileMD5), + let firstResult = results.first, + let systemID = firstResult["systemID"] as? Int, + let system = possibleSystems.first(where: { $0.openvgDatabaseID == systemID }) { + ILOG("System determined by MD5 match in OpenVGDB: \(system.name)") + return system + } + } catch { + ELOG("Error searching OpenVGDB by MD5: \(error.localizedDescription)") + } + } + + // If still no match, try to determine based on file content + // This is a placeholder for more advanced content-based detection + // You might want to implement system-specific logic here + for system in possibleSystems { + if doesFileContentMatch(candidate, forSystem: system) { + ILOG("System determined by file content match: \(system.name)") + return system + } + } + + // If we still couldn't determine the system, return the first possible system as a fallback + WLOG("Could not determine system from content, using first possible system as fallback") + return possibleSystems[0] + } + + /// Checks if a file content matches a given system + private func doesFileContentMatch(_ candidate: ImportCandidateFile, forSystem system: PVSystem) -> Bool { + // Implement system-specific file content matching logic here + // This could involve checking file headers, file structure, or other system-specific traits + // For now, we'll return false as a placeholder + return false + } + + /// Determines the system for a given candidate file + internal func determineSystem(for candidate: ImportCandidateFile) async throws -> PVSystem { + guard let md5 = candidate.md5?.uppercased() else { + throw GameImporterError.couldNotCalculateMD5 + } + + let fileExtension = candidate.filePath.pathExtension.lowercased() + + DLOG("Checking MD5: \(md5) for possible BIOS match") + // First check if this is a BIOS file by MD5 + let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) + if !biosMatches.isEmpty { + DLOG("Found BIOS matches: \(biosMatches.map { $0.expectedFilename }.joined(separator: ", "))") + // Copy BIOS to all matching system directories + for bios in biosMatches { + if let system = bios.system { + DLOG("Copying BIOS to system: \(system.name)") + let biosPath = PVEmulatorConfiguration.biosPath(forSystemIdentifier: system.identifier) + .appendingPathComponent(bios.expectedFilename) + try FileManager.default.copyItem(at: candidate.filePath, to: biosPath) + } + } + // Return the first system that uses this BIOS + if let firstSystem = biosMatches.first?.system { + DLOG("Using first matching system for BIOS: \(firstSystem.name)") + return firstSystem + } + } + + // Check if it's a CD-based game first + if PVEmulatorConfiguration.supportedCDFileExtensions.contains(fileExtension) { + if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + if systems.count == 1 { + return systems[0] + } else if systems.count > 1 { + // For CD games with multiple possible systems, use content detection + return try determineSystemFromContent(for: candidate, possibleSystems: systems) + } + } + } + + // Try to find system by MD5 using OpenVGDB + if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), + let firstResult = results.first, + let systemID = firstResult["systemID"] as? NSNumber { + + // Get all matching systems + let matchingSystems = results.compactMap { result -> PVSystem? in + guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } + return PVEmulatorConfiguration.system(forIdentifier: sysID) + } + + if !matchingSystems.isEmpty { + // Sort by release year and take the oldest + if let oldestSystem = matchingSystems.sorted(by: { $0.releaseYear < $1.releaseYear }).first { + DLOG("System determined by MD5 match (oldest): \(oldestSystem.name) (\(oldestSystem.releaseYear))") + return oldestSystem + } + } + + // Fallback to original single system match if sorting fails + if let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { + DLOG("System determined by MD5 match (fallback): \(system.name)") + return system + } + } + + DLOG("MD5 lookup failed, trying filename matching") + + // Try filename matching next + let fileName = candidate.filePath.lastPathComponent + + if let matchedSystem = await matchSystemByFileName(fileName) { + DLOG("Found system by filename match: \(matchedSystem.name)") + return matchedSystem + } + + let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: candidate.filePath.pathExtension.lowercased()) ?? [] + + // If MD5 lookup fails, try to determine the system based on file extension + if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + if systems.count == 1 { + return systems[0] + } else if systems.count > 1 { + // If multiple systems support this extension, try to determine based on file content or metadata + return try await determineSystemFromContent(for: candidate, possibleSystems: systems) + } + } + + throw GameImporterError.noSystemMatched + } + + /// Retrieves the system ID from the cache for a given ROM candidate + public func systemIdFromCache(forROMCandidate rom: ImportCandidateFile) -> String? { + guard let md5 = rom.md5 else { + ELOG("MD5 was blank") + return nil + } + if let result = RomDatabase.artMD5DBCache[md5] ?? RomDatabase.getArtCacheByFileName(rom.filePath.lastPathComponent), + let databaseID = result["systemID"] as? Int, + let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { + return systemID + } + return nil + } + + /// Matches a system based on the ROM candidate + public func systemId(forROMCandidate rom: ImportCandidateFile) -> String? { + guard let md5 = rom.md5 else { + ELOG("MD5 was blank") + return nil + } + + let fileName: String = rom.filePath.lastPathComponent + + do { + if let databaseID = try openVGDB.system(forRomMD5: md5, or: fileName), + let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { + return systemID + } else { + ILOG("Could't match \(rom.filePath.lastPathComponent) based off of MD5 {\(md5)}") + return nil + } + } catch { + DLOG("Unable to find rom by MD5: \(error.localizedDescription)") + return nil + } + } + + internal func determineSystemByMD5(_ candidate: ImportCandidateFile) async throws -> PVSystem? { + guard let md5 = candidate.md5?.uppercased() else { + throw GameImporterError.couldNotCalculateMD5 + } + + DLOG("Attempting MD5 lookup for: \(md5)") + + // Try to find system by MD5 using OpenVGDB + if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), + let firstResult = results.first, + let systemID = firstResult["systemID"] as? NSNumber, + let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { + DLOG("System determined by MD5 match: \(system.name)") + return system + } + + DLOG("No system found by MD5") + return nil + } + + /// Determines the systems for a given path + internal func determineSystems(for path: URL, chosenSystem: System?) throws -> [PVSystem] { + if let chosenSystem = chosenSystem { + if let system = RomDatabase.systemCache[chosenSystem.identifier] { + return [system] + } + } + + let fileExtensionLower = path.pathExtension.lowercased() + return PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtensionLower) ?? [] + } + + /// Finds any current game that could belong to any of the given systems + internal class func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? { + // Check if existing ROM + + let allGames = RomDatabase.gamesCache.values.filter ({ + $0.romPath.lowercased() == romFilename.lowercased() + }) + /* + let database = RomDatabase.sharedInstance + + let predicate = NSPredicate(format: "romPath CONTAINS[c] %@", PVEmulatorConfiguration.stripDiscNames(fromFilename: romFilename)) + let allGames = database.all(PVGame.self, filter: predicate) + */ + // Optionally filter to specfici systems + if let systems = systems { + //let filteredGames = allGames.filter { systems.contains($0.system) } + var sysIds:[String:Bool]=[:] + systems.forEach({ sysIds[$0.identifier] = true }) + let filteredGames = allGames.filter { sysIds[$0.systemIdentifier] != nil } + return filteredGames.isEmpty ? nil : Array(filteredGames) + } else { + return allGames.isEmpty ? nil : Array(allGames) + } + } + + /// Returns the system identifiers for a given ROM path + public func systemIDsForRom(at path: URL) -> [String]? { + let fileExtension: String = path.pathExtension.lowercased() + return romExtensionToSystemsMap[fileExtension] + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift new file mode 100644 index 0000000000..cca3c1b05e --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift @@ -0,0 +1,59 @@ +// +// GameImporter+Utils.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + + +extension GameImporter { + + /// Calculates the MD5 hash for a given game + @objc + public func calculateMD5(forGame game: PVGame) -> String? { + var offset: UInt64 = 0 + + if game.systemIdentifier == SystemIdentifier.SNES.rawValue { + offset = SystemIdentifier.SNES.offset + } else if let system = SystemIdentifier(rawValue: game.systemIdentifier) { + offset = system.offset + } + + let romPath = romsPath.appendingPathComponent(game.romPath, isDirectory: false) + let fm = FileManager.default + if !fm.fileExists(atPath: romPath.path) { + ELOG("Cannot find file at path: \(romPath)") + return nil + } + + return fm.md5ForFile(atPath: romPath.path, fromOffset: offset) + } + + /// Saves the relative path for a given game + func saveRelativePath(_ existingGame: PVGame, partialPath:String, file:URL) { + Task { + if await RomDatabase.gamesCache[partialPath] == nil { + await RomDatabase.addRelativeFileCache(file, game:existingGame) + } + } + } + + /// Checks if a given ROM file is a CD-ROM + internal func isCDROM(_ romFile: ImportCandidateFile) -> Bool { + return isCDROM(romFile.filePath) + } + + /// Checks if a given path is a CD-ROM + internal func isCDROM(_ path: URL) -> Bool { + let cdromExtensions: Set = Extensions.discImageExtensions.union(Extensions.playlistExtensions) + let fileExtension = path.pathExtension.lowercased() + return cdromExtensions.contains(fileExtension) + } + + /// Checks if a given path is artwork + package func isArtwork(_ path: URL) -> Bool { + let artworkExtensions = Extensions.artworkExtensions + let fileExtension = path.pathExtension.lowercased() + return artworkExtensions.contains(fileExtension) + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index dcc1dcc111..4f3b55e074 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -68,7 +68,7 @@ import AppKit */ /// Import Coodinator -private actor ImportCoordinator { +internal actor ImportCoordinator { private var activeImports: Set = [] func checkAndRegisterImport(md5: String) -> Bool { @@ -103,63 +103,7 @@ public typealias GameImporterFinishedImportingGameHandler = (_ md5Hash: String, public typealias GameImporterFinishedGettingArtworkHandler = (_ artworkURL: String?) -> Void -// Enum to define the possible statuses of each import -public enum ImportStatus: String { - case queued - case processing - case success - case failure - case conflict // Indicates additional action needed by user after successful import - - public var description: String { - switch self { - case .queued: return "Queued" - case .processing: return "Processing" - case .success: return "Completed" - case .failure: return "Failed" - case .conflict: return "Conflict" - } - } - - public var color: Color { - switch self { - case .queued: return .gray - case .processing: return .blue - case .success: return .green - case .failure: return .red - case .conflict: return .yellow - } - } -} - -// Enum to define file types for each import -public enum FileType { - case bios, artwork, game, cdRom, unknown -} - -// Enum to track processing state -public enum ProcessingState { - case idle - case processing -} -// ImportItem model to hold each file's metadata and progress -@Observable -public class ImportItem: Identifiable, ObservableObject { - public let id = UUID() - public let url: URL - public let fileType: FileType - public let system: String // Can be set to the specific system type - - // Observable status for individual imports - public var status: ImportStatus = .queued - - public init(url: URL, fileType: FileType, system: String) { - self.url = url - self.fileType = fileType - self.system = system - } -} @@ -178,7 +122,7 @@ public final class GameImporter: ObservableObject { /// Closure called when artwork finishes downloading public var finishedArtworkHandler: GameImporterFinishedGettingArtworkHandler? /// Flag indicating if conflicts were encountered during import - public private(set) var encounteredConflicts = false + public internal(set) var encounteredConflicts = false /// Spotlight Handerls /// Closure called when spotlight completes @@ -209,82 +153,17 @@ public final class GameImporter: ObservableObject { }() /// Map of system identifiers to their ROM paths - public private(set) var systemToPathMap = [String: URL]() + public internal(set) var systemToPathMap = [String: URL]() /// Map of ROM extensions to their corresponding system identifiers - public private(set) var romExtensionToSystemsMap = [String: [String]]() + public internal(set) var romExtensionToSystemsMap = [String: [String]]() // MARK: - Queue public var importStatus: String = "" - public var importQueue: [ImportItem] = [] + public var importQueue: [ImportQueueItem] = [] public var processingState: ProcessingState = .idle // Observable state for processing status - - // Adds an ImportItem to the queue without starting processing - public func addImport(_ item: ImportItem) { - importQueue.append(item) - - startProcessing() - } - - // Public method to manually start processing if needed - public func startProcessing() { - // Only start processing if it's not already active - guard processingState == .idle else { return } - Task { - await processQueue() - } - } - - // Processes each ImportItem in the queue sequentially - private func processQueue() async { - DispatchQueue.main.async { - self.processingState = .processing - } - - for item in importQueue where item.status == .queued { - await processItem(item) - } - - DispatchQueue.main.async { - self.processingState = .idle // Reset processing status when queue is fully processed - } - } - - // Process a single ImportItem and update its status - private func processItem(_ item: ImportItem) async { - item.status = .processing - updateStatus("Importing \(item.url.lastPathComponent) for \(item.system)") - - do { - // Simulate file processing - try await performImport(for: item) - item.status = .success - updateStatus("Completed \(item.url.lastPathComponent) for \(item.system)") - } catch { - if error.localizedDescription.contains("Conflict") { - item.status = .conflict - updateStatus("Conflict for \(item.url.lastPathComponent). User action needed.") - } else { - item.status = .failure - updateStatus("Failed \(item.url.lastPathComponent) with error: \(error.localizedDescription)") - } - } - } - - private func performImport(for item: ImportItem) async throws { - try await Task.sleep(nanoseconds: 5_000_000_000) - } - - // General status update for GameImporter - private func updateStatus(_ message: String) { - DispatchQueue.main.async { - self.importStatus = message - } - } - - // MARK: - Paths @@ -305,31 +184,6 @@ public final class GameImporter: ObservableObject { return systemToPathMap[systemID] } - /// Returns the system identifiers for a given ROM path - public func systemIDsForRom(at path: URL) -> [String]? { - let fileExtension: String = path.pathExtension.lowercased() - return romExtensionToSystemsMap[fileExtension] - } - - /// Checks if a given ROM file is a CD-ROM - internal func isCDROM(_ romFile: ImportCandidateFile) -> Bool { - return isCDROM(romFile.filePath) - } - - /// Checks if a given path is a CD-ROM - private func isCDROM(_ path: URL) -> Bool { - let cdromExtensions: Set = Extensions.discImageExtensions.union(Extensions.playlistExtensions) - let fileExtension = path.pathExtension.lowercased() - return cdromExtensions.contains(fileExtension) - } - - /// Checks if a given path is artwork - private func isArtwork(_ path: URL) -> Bool { - let artworkExtensions = Extensions.artworkExtensions - let fileExtension = path.pathExtension.lowercased() - return artworkExtensions.contains(fileExtension) - } - /// Bundle for this module fileprivate let ThisBundle: Bundle = Bundle.module /// Token for notifications @@ -337,7 +191,7 @@ public final class GameImporter: ObservableObject { /// DispatchGroup for initialization public let initialized = DispatchGroup() - private let importCoordinator = ImportCoordinator() + internal let importCoordinator = ImportCoordinator() /// Initializes the GameImporter fileprivate init() { @@ -434,1537 +288,132 @@ public final class GameImporter: ObservableObject { deinit { notificationToken?.invalidate() } -} - -// MARK: - Importing Functions - -extension GameImporter { - /// Imports files from given paths - public func importFiles(atPaths paths: [URL]) async throws -> [URL] { - let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) - var importedFiles: [URL] = [] - - for path in sortedPaths { - do { - if let importedFile = try await importSingleFile(at: path) { - importedFiles.append(importedFile) - } - } catch { - ELOG("Failed to import file at \(path.path): \(error.localizedDescription)") - } - } - - return importedFiles - } - /// Imports a single file from the given path - private func importSingleFile(at path: URL) async throws -> URL? { - guard FileManager.default.fileExists(atPath: path.path) else { - WLOG("File doesn't exist at \(path.path)") - return nil - } + //MARK: Queue Management + + // Adds an ImportItem to the queue without starting processing + public func addImport(_ item: ImportQueueItem) { + addImportItemToQueue(item) - if isCDROM(path) { - return try await handleCDROM(at: path) - } else if isArtwork(path) { - return try await handleArtwork(at: path) - } else { - return try await handleROM(at: path) - } + startProcessing() } - /// Handles importing a CD-ROM - private func handleCDROM(at path: URL) async throws -> URL? { - let movedToPaths = try await moveCDROM(toAppropriateSubfolder: ImportCandidateFile(filePath: path)) - if let movedToPaths = movedToPaths { - let pathsString = movedToPaths.map { $0.path }.joined(separator: ", ") - VLOG("Found a CD. Moved files to the following paths \(pathsString)") - } - return nil + public func addImports(forPaths paths: [URL]) { + paths.forEach({ (url) in + addImportItemToQueue(ImportQueueItem(url: url, fileType: .unknown, system: "")) + }) + + startProcessing() } - - /// Handles importing artwork - private func handleArtwork(at path: URL) async throws -> URL? { - if let game = await GameImporter.importArtwork(fromPath: path) { - ILOG("Found artwork \(path.lastPathComponent) for game <\(game.title)>") + + // Public method to manually start processing if needed + public func startProcessing() { + // Only start processing if it's not already active + guard processingState == .idle else { return } + Task { + await processQueue() } - return nil - } - - /// Handles importing a ROM - private func handleROM(at path: URL) async throws -> URL? { - let candidate = ImportCandidateFile(filePath: path) - return try await moveROM(toAppropriateSubfolder: candidate) } - - /// Starts an import for the given paths - public func startImport(forPaths paths: [URL]) async { - // Pre-sort - let paths = PVEmulatorConfiguration.sortImportURLs(urls: paths) - let scanOperation = BlockOperation { - Task { - do { - let newPaths = try await self.importFiles(atPaths: paths) - await self.getRomInfoForFiles(atPaths: newPaths, userChosenSystem: nil) - } catch { - ELOG("\(error)") - } - } + + // Processes each ImportItem in the queue sequentially + private func processQueue() async { + ILOG("GameImportQueue - processQueue beginning Import Processing") + DispatchQueue.main.async { + self.processingState = .processing } - let completionOperation = BlockOperation { - if self.completionHandler != nil { - DispatchQueue.main.sync(execute: { () -> Void in - self.completionHandler?(self.encounteredConflicts) - }) - } + for item in importQueue where item.status == .queued { + await processItem(item) } - completionOperation.addDependency(scanOperation) - serialImportQueue.addOperation(scanOperation) - serialImportQueue.addOperation(completionOperation) + DispatchQueue.main.async { + self.processingState = .idle // Reset processing status when queue is fully processed + } + ILOG("GameImportQueue - processQueue complete Import Processing") } -} -// MARK: - Moving Functions + // Process a single ImportItem and update its status + private func processItem(_ item: ImportQueueItem) async { + ILOG("GameImportQueue - processing item in queue: \(item.url)") + item.status = .processing + updateImporterStatus("Importing \(item.url.lastPathComponent) for \(item.system)") -extension GameImporter { - /// Moves a CD-ROM to the appropriate subfolder - private func moveCDROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> [URL]? { - guard isCDROM(candidate.filePath) else { - return nil - } - - let fileManager = FileManager.default - let fileName = candidate.filePath.lastPathComponent - - guard let system = try? await determineSystem(for: candidate) else { - throw GameImporterError.unsupportedSystem - } - - let destinationFolder = system.romsDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - do { - try fileManager.createDirectory(at: destinationFolder, withIntermediateDirectories: true, attributes: nil) - try fileManager.moveItem(at: candidate.filePath, to: destinationPath) - let relatedFiles = try await moveRelatedFiles(for: candidate, to: destinationFolder) - return [destinationPath] + relatedFiles + // Simulate file processing + try await performImport(for: item) + item.status = .success + updateImporterStatus("Completed \(item.url.lastPathComponent) for \(item.system)") + ILOG("GameImportQueue - processing item in queue: \(item.url) completed.") } catch { - throw GameImporterError.failedToMoveCDROM(error) - } - } - - /// Moves a ROM to the appropriate subfolder - private func moveROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> URL? { - guard !isCDROM(candidate.filePath) else { - return nil - } - - let fileManager = FileManager.default - let fileName = candidate.filePath.lastPathComponent - - // Check first if known BIOS - if let system = try await handleBIOSFile(candidate) { - DLOG("Moving BIOS file to system: \(system.name)") - let destinationFolder = system.biosDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - return try await moveROMFile(candidate, to: destinationPath) - } - - // Check for M3U - if let system = try await handleM3UFile(candidate) { - DLOG("Moving M3U and referenced files to system: \(system.name)") - // Move M3U and all referenced files to system directory - let destinationDir = system.romsDirectory - return try await moveM3UAndReferencedFiles(candidate, to: destinationDir) - } - - // CD-ROM handling - if let system = try await handleCDROMFile(candidate) { - DLOG("Moving CD-ROM files to system: \(system.name)") - let destinationDir = system.romsDirectory - return try await moveCDROMFiles(candidate, to: destinationDir) - } - - // Regular ROM handling - let (system, hasConflict) = try await handleRegularROM(candidate) - let destinationDir = hasConflict ? self.conflictPath : system.romsDirectory - - DLOG("Moving ROM to \(hasConflict ? "conflicts" : "system") directory: \(system.name)") - return try await moveROMFile(candidate, to: destinationDir) - } - - private func handleBIOSFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - guard let md5 = candidate.md5?.uppercased() else { - return nil - } - - // Get all BIOS entries that match this MD5 - let matchingBIOSEntries = PVEmulatorConfiguration.biosEntries.filter { biosEntry in - let frozenBiosEntry = biosEntry.isFrozen ? biosEntry : biosEntry.freeze() - return frozenBiosEntry.expectedMD5.uppercased() == md5 - } - - if !matchingBIOSEntries.isEmpty { - // Get the first matching system - if let firstBIOSEntry = matchingBIOSEntries.first { - let frozenBiosEntry = firstBIOSEntry.isFrozen ? firstBIOSEntry : firstBIOSEntry.freeze() - - // Move file to BIOS directory - let destinationURL = frozenBiosEntry.expectedPath - try await moveROMFile(candidate, to: destinationURL) - - // Update BIOS entry in Realm - try await MainActor.run { - let realm = try Realm() - try realm.write { - if let thawedBios = frozenBiosEntry.thaw() { - let biosFile = PVFile(withURL: destinationURL) - thawedBios.file = biosFile - } - } - } - - return frozenBiosEntry.system - } - } - - return nil - } - - private func handleCDROMFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - let `extension` = candidate.filePath.pathExtension.lowercased() - guard PVEmulatorConfiguration.supportedCDFileExtensions.contains(`extension`) else { - return nil - } - - DLOG("Handling CD-ROM file: \(candidate.filePath.lastPathComponent)") - - // First try MD5 matching - if let system = try? await determineSystem(for: candidate) { - DLOG("Found system match for CD-ROM by MD5: \(system.name)") - return system - } - - // If cue file, try to match its bin file - if `extension` == "cue" { - if let binFile = try findAssociatedBinFile(for: candidate) { - DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") - let binCandidate = ImportCandidateFile(filePath: binFile) - if let system = try? await determineSystem(for: binCandidate) { - DLOG("Found system match from associated bin file: \(system.name)") - return system - } + ILOG("GameImportQueue - processing item in queue: \(item.url) caught error...") + if error.localizedDescription.contains("Conflict") { + item.status = .conflict + updateImporterStatus("Conflict for \(item.url.lastPathComponent). User action needed.") + WLOG("GameImportQueue - processing item in queue: \(item.url) restuled in conflict.") + } else { + item.status = .failure + updateImporterStatus("Failed \(item.url.lastPathComponent) with error: \(error.localizedDescription)") + ELOG("GameImportQueue - processing item in queue: \(item.url) restuled in error: \(error.localizedDescription)") } } - - // Try exact filename match - if let system = await matchSystemByFileName(candidate.filePath.lastPathComponent) { - DLOG("Found system match by filename: \(system.name)") - return system - } - - DLOG("No system match found for CD-ROM file") - return nil - } - - /// Move a `ImportCandidateFile` to a destination, creating the destination directory if needed - private func moveROMFile(_ romFile: ImportCandidateFile, to destination: URL) async throws -> URL { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let destPath = destination.appendingPathComponent(romFile.filePath.lastPathComponent) - try FileManager.default.moveItem(at: romFile.filePath, to: destPath) - DLOG("Moved ROM file to: \(destPath.path)") - return destPath } - - private func findAssociatedBinFile(for cueFile: ImportCandidateFile) throws -> URL? { - let cueContents = try String(contentsOf: cueFile.filePath, encoding: .utf8) - let lines = cueContents.components(separatedBy: .newlines) + + private func performImport(for item: ImportQueueItem) async throws { - // Look for FILE "something.bin" BINARY line - for line in lines { - let components = line.trimmingCharacters(in: .whitespaces) - .components(separatedBy: "\"") - guard components.count >= 2, - line.lowercased().contains("file") && line.lowercased().contains("binary") else { - continue - } - - let binFileName = components[1] - let binPath = cueFile.filePath.deletingLastPathComponent().appendingPathComponent(binFileName) - - if FileManager.default.fileExists(atPath: binPath.path) { - return binPath - } + //detect type for updating UI + //todo: detect BIOS + if (isCDROM(item.url)) { + item.fileType = .cdRom + } else if (isArtwork(item.url)) { + item.fileType = .artwork + } else { + item.fileType = .game } - return nil - } - - private func moveM3UAndReferencedFiles(_ m3uFile: ImportCandidateFile, to destination: URL) async throws -> URL { - let contents = try String(contentsOf: m3uFile.filePath, encoding: .utf8) - let files = contents.components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty && !$0.hasPrefix("#") } - - // Create destination directory if needed - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + var importedFiles: [URL] = [] - // Move all referenced files - for file in files { - let sourcePath = m3uFile.filePath.deletingLastPathComponent().appendingPathComponent(file) - let destPath = destination.appendingPathComponent(file) - - if FileManager.default.fileExists(atPath: sourcePath.path) { - try FileManager.default.moveItem(at: sourcePath, to: destPath) - DLOG("Moved M3U referenced file: \(file)") + do { + if let importedFile = try await importSingleFile(at: item.url) { + importedFiles.append(importedFile) } + } catch { + //TODO: what do i do here? + ELOG("Failed to import file at \(item.url): \(error.localizedDescription)") } - // Move M3U file itself - let m3uDestPath = destination.appendingPathComponent(m3uFile.filePath.lastPathComponent) - try FileManager.default.moveItem(at: m3uFile.filePath, to: m3uDestPath) - DLOG("Moved M3U file to: \(m3uDestPath.path)") - - return m3uDestPath - } - - private func moveCDROMFiles(_ cdFile: ImportCandidateFile, to destination: URL) async throws -> URL { - let fileManager = FileManager.default - try fileManager.createDirectory(at: destination, withIntermediateDirectories: true) - - let `extension` = cdFile.filePath.pathExtension.lowercased() - let destPath = destination.appendingPathComponent(cdFile.filePath.lastPathComponent) - - // If it's a cue file, move both cue and bin - if `extension` == "cue" { - if let binPath = try findAssociatedBinFile(for: cdFile) { - let binDestPath = destination.appendingPathComponent(binPath.lastPathComponent) - try fileManager.moveItem(at: binPath, to: binDestPath) - DLOG("Moved bin file to: \(binDestPath.path)") + await importedFiles.asyncForEach { path in + do { + try await self._handlePath(path: path, userChosenSystem: nil) + } catch { + //TODO: what do i do here? I could just let this throw or try and process what happened... + ELOG("\(error)") } - } - - // Move the main CD-ROM file - try fileManager.moveItem(at: cdFile.filePath, to: destPath) - DLOG("Moved CD-ROM file to: \(destPath.path)") - - return destPath - } - - /// Moves related files for a given candidate - private func moveRelatedFiles(for candidate: ImportCandidateFile, to destinationFolder: URL) async throws -> [URL] { - let fileManager = FileManager.default - let fileName = candidate.filePath.deletingPathExtension().lastPathComponent - let sourceFolder = candidate.filePath.deletingLastPathComponent() - - let relatedFiles = try fileManager.contentsOfDirectory(at: sourceFolder, includingPropertiesForKeys: nil) - .filter { $0.deletingPathExtension().lastPathComponent == fileName && $0 != candidate.filePath } + } // for each - return try await withThrowingTaskGroup(of: URL.self) { group in - for file in relatedFiles { - group.addTask { - let destination = destinationFolder.appendingPathComponent(file.lastPathComponent) - try fileManager.moveItem(at: file, to: destination) - return destination - } - } - - var movedFiles: [URL] = [] - for try await movedFile in group { - movedFiles.append(movedFile) - } - return movedFiles - } + //external callers - might not be needed in the end + self.completionHandler?(self.encounteredConflicts) } - - /// Moves a file and overwrites if it already exists at the destination - public func moveAndOverWrite(sourcePath: URL, destinationPath: URL) throws { - let fileManager = FileManager.default - - // If file exists at destination, remove it first - if fileManager.fileExists(atPath: destinationPath.path) { - try fileManager.removeItem(at: destinationPath) + + // General status update for GameImporter + internal func updateImporterStatus(_ message: String) { + DispatchQueue.main.async { + self.importStatus = message } - - // Now move the file - try fileManager.moveItem(at: sourcePath, to: destinationPath) } - /// BIOS entry matching - private func biosEntryMatching(candidateFile: ImportCandidateFile) -> [PVBIOS]? { - let fileName = candidateFile.filePath.lastPathComponent - var matchingBioses = Set() - - DLOG("Checking if file is BIOS: \(fileName)") - - // First try to match by filename - if let biosEntry = PVEmulatorConfiguration.biosEntry(forFilename: fileName) { - DLOG("Found BIOS match by filename: \(biosEntry.expectedFilename)") - matchingBioses.insert(biosEntry) - } - - // Then try to match by MD5 - if let md5 = candidateFile.md5?.uppercased(), - let md5Entry = PVEmulatorConfiguration.biosEntry(forMD5: md5) { - DLOG("Found BIOS match by MD5: \(md5Entry.expectedFilename)") - matchingBioses.insert(md5Entry) + private func addImportItemToQueue(_ item: ImportQueueItem) { + let duplicate = importQueue.contains { existing in + return (existing.url == item.url || existing.id == item.id) ? true : false } - if !matchingBioses.isEmpty { - let matches = Array(matchingBioses) - DLOG("Found \(matches.count) BIOS matches") - return matches + guard !duplicate else { + WLOG("GameImportQueue - Trying to add duplicate ImportItem to import queue with url: \(item.url) and id: \(item.id)") + return; } - return nil + importQueue.append(item) + ILOG("GameImportQueue - add ImportItem to import queue with url: \(item.url) and id: \(item.id)") } } -// MARK: - Conflict Resolution -extension GameImporter { - /// Resolves conflicts with given solutions - public func resolveConflicts(withSolutions solutions: [URL: System]) async { - let importOperation = BlockOperation() - - await solutions.asyncForEach { filePath, system in - let subfolder = system.romsDirectory - - if !FileManager.default.fileExists(atPath: subfolder.path, isDirectory: nil) { - ILOG("Path <\(subfolder.path)> doesn't exist. Creating.") - do { - try FileManager.default.createDirectory(at: subfolder, withIntermediateDirectories: true, attributes: nil) - } catch { - ELOG("Error making conflicts dir <\(subfolder.path)>") - assertionFailure("Error making conflicts dir <\(subfolder.path)>") - } - } - - let sourceFilename: String = filePath.lastPathComponent - let sourcePath: URL = filePath - let destinationPath: URL = subfolder.appendingPathComponent(sourceFilename, isDirectory: false) - - do { - try moveAndOverWrite(sourcePath: sourcePath, destinationPath: destinationPath) - } catch { - ELOG("\(error)") - } - - let relatedFileName: String = sourcePath.deletingPathExtension().lastPathComponent - - let conflictsDirContents = try? FileManager.default.contentsOfDirectory(at: conflictPath, includingPropertiesForKeys: nil, options: []) - conflictsDirContents?.forEach { file in - var fileWithoutExtension: String = file.deletingPathExtension().lastPathComponent - fileWithoutExtension = PVEmulatorConfiguration.stripDiscNames(fromFilename: fileWithoutExtension) - let relatedFileName = PVEmulatorConfiguration.stripDiscNames(fromFilename: relatedFileName) - - if fileWithoutExtension == relatedFileName { - let isCueSheet = destinationPath.pathExtension == "cue" - - if isCueSheet { - let cueSheetPath = destinationPath - if var cuesheetContents = try? String(contentsOf: cueSheetPath, encoding: .utf8) { - let range = (cuesheetContents as NSString).range(of: file.lastPathComponent, options: .caseInsensitive) - - if range.location != NSNotFound { - if let subRange = Range(range, in: cuesheetContents) { - cuesheetContents.replaceSubrange(subRange, with: file.lastPathComponent) - } - - do { - try cuesheetContents.write(to: cueSheetPath, atomically: true, encoding: .utf8) - } catch { - ELOG("Unable to rewrite cuesheet \(destinationPath.path) because \(error.localizedDescription)") - } - } else { - DLOG("Range of string <\(file)> not found in file <\(cueSheetPath.lastPathComponent)>") - } - } - } - - do { - let newDestinationPath = subfolder.appendingPathComponent(file.lastPathComponent, isDirectory: false) - try moveAndOverWrite(sourcePath: file, destinationPath: newDestinationPath) - NSLog("Moving \(file.lastPathComponent) to \(newDestinationPath)") - } catch { - ELOG("Unable to move related file from \(filePath.path) to \(subfolder.path) because: \(error.localizedDescription)") - } - } - } - - importOperation.addExecutionBlock { - Task { - ILOG("Import Files at \(destinationPath)") - if let system = RomDatabase.systemCache[system.identifier] { - RomDatabase.addFileSystemROMCache(system) - } - await self.getRomInfoForFiles(atPaths: [destinationPath], userChosenSystem: system) - } - } - } - - let completionOperation = BlockOperation { - if self.completionHandler != nil { - DispatchQueue.main.async(execute: { () -> Void in - self.completionHandler?(false) - }) - } - } - - completionOperation.addDependency(importOperation) - serialImportQueue.addOperation(importOperation) - serialImportQueue.addOperation(completionOperation) - } -} - -// MARK: - Artwork Handling - -public extension GameImporter { - /// Imports artwork from a given path - class func importArtwork(fromPath imageFullPath: URL) async -> PVGame? { - var isDirectory: ObjCBool = false - let fileExists = FileManager.default.fileExists(atPath: imageFullPath.path, isDirectory: &isDirectory) - if !fileExists || isDirectory.boolValue { - WLOG("File doesn't exist or is directory at \(imageFullPath)") - return nil - } - - var success = false - - defer { - if success { - do { - try FileManager.default.removeItem(at: imageFullPath) - } catch { - ELOG("Failed to delete image at path \(imageFullPath) \n \(error.localizedDescription)") - } - } - } - - let gameFilename: String = imageFullPath.deletingPathExtension().lastPathComponent - let gameExtension = imageFullPath.deletingPathExtension().pathExtension - let database = RomDatabase.sharedInstance - - if gameExtension.isEmpty { - ILOG("Trying to import artwork that didn't contain the extension of the system") - let games = database.all(PVGame.self, filter: NSPredicate(format: "romPath CONTAINS[c] %@", argumentArray: [gameFilename])) - - if games.count == 1, let game = games.first { - ILOG("File for image didn't have extension for system but we found a single match for image \(imageFullPath.lastPathComponent) to game \(game.title) on system \(game.systemIdentifier)") - guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { - return nil - } - - do { - try database.writeTransaction { - game.customArtworkURL = hash - } - success = true - ILOG("Set custom artwork of game \(game.title) from file \(imageFullPath.lastPathComponent)") - } catch { - ELOG("Couldn't update game \(game.title) with new artwork URL \(hash)") - } - - return game - } else { - VLOG("Database search returned \(games.count) results") - } - } - - guard let systems: [PVSystem] = PVEmulatorConfiguration.systemsFromCache(forFileExtension: gameExtension), !systems.isEmpty else { - ELOG("No system for extension \(gameExtension)") - return nil - } - - let cdBasedSystems = PVEmulatorConfiguration.cdBasedSystems - let couldBelongToCDSystem = !Set(cdBasedSystems).isDisjoint(with: Set(systems)) - - if (couldBelongToCDSystem && PVEmulatorConfiguration.supportedCDFileExtensions.contains(gameExtension.lowercased())) || systems.count > 1 { - guard let existingGames = findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: gameFilename) else { - ELOG("System for extension \(gameExtension) is a CD system and {\(gameExtension)} not the right matching file type of cue or m3u") - return nil - } - if existingGames.count == 1, let onlyMatch = existingGames.first { - ILOG("We found a hit for artwork that could have been belonging to multiple games and only found one file that matched by systemid/filename. The winner is \(onlyMatch.title) for \(onlyMatch.systemIdentifier)") - - guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { - ELOG("Couldn't move image, fail to set custom artwork") - return nil - } - - do { - try database.writeTransaction { - onlyMatch.customArtworkURL = hash - } - success = true - } catch { - ELOG("Couldn't update game \(onlyMatch.title) with new artwork URL") - } - return onlyMatch - } else { - ELOG("We got to the unlikely scenario where an extension is possibly a CD binary file, or belongs to a system, and had multiple games that matched the filename under more than one core.") - return nil - } - } - - guard let system = systems.first else { - ELOG("systems empty") - return nil - } - - var gamePartialPath: String = URL(fileURLWithPath: system.identifier, isDirectory: true).appendingPathComponent(gameFilename).deletingPathExtension().path - if gamePartialPath.first == "/" { - gamePartialPath.removeFirst() - } - - if gamePartialPath.isEmpty { - ELOG("Game path was empty") - return nil - } - - var games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), value: gamePartialPath) - if games.isEmpty { - games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), beginsWith: gamePartialPath) - } - - guard !games.isEmpty else { - ELOG("Couldn't find game for path \(gamePartialPath)") - return nil - } - - if games.count > 1 { - WLOG("There were multiple matches for \(gamePartialPath)! #\(games.count). Going with first for now until we make better code to prompt user.") - } - - let game = games.first! - - guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { - ELOG("scaleAndMoveImageToCache failed") - return nil - } - - do { - try database.writeTransaction { - game.customArtworkURL = hash - } - success = true - } catch { - ELOG("Couldn't update game with new artwork URL") - } - - return game - } - - /// Scales and moves an image to the cache - fileprivate class func scaleAndMoveImageToCache(imageFullPath: URL) -> String? { - let coverArtFullData: Data - do { - coverArtFullData = try Data(contentsOf: imageFullPath, options: []) - } catch { - ELOG("Couldn't read data from image file \(imageFullPath.path)\n\(error.localizedDescription)") - return nil - } - -#if canImport(UIKit) - guard let coverArtFullImage = UIImage(data: coverArtFullData) else { - ELOG("Failed to create Image from data") - return nil - } - guard let coverArtScaledImage = coverArtFullImage.scaledImage(withMaxResolution: Int(PVThumbnailMaxResolution)) else { - ELOG("Failed to create scale image") - return nil - } -#else - guard let coverArtFullImage = NSImage(data: coverArtFullData) else { - ELOG("Failed to create Image from data") - return nil - } - let coverArtScaledImage = coverArtFullImage -#endif - -#if canImport(UIKit) - guard let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) else { - ELOG("Failed to create data representation of scaled image") - return nil - } -#else - let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) -#endif - - let hash: String = (coverArtScaledData as NSData).md5 - - do { - let destinationURL = try PVMediaCache.writeData(toDisk: coverArtScaledData, withKey: hash) - VLOG("Scaled and moved image from \(imageFullPath.path) to \(destinationURL.path)") - } catch { - ELOG("Failed to save artwork to cache: \(error.localizedDescription)") - return nil - } - - return hash - } -} - -// MARK: - System Management - -extension GameImporter { - - private func matchSystemByPartialName(_ fileName: String, possibleSystems: [PVSystem]) -> PVSystem? { - let cleanedName = fileName.lowercased() - - for system in possibleSystems { - let patterns = filenamePatterns(forSystem: system) - - for pattern in patterns { - if (try? NSRegularExpression(pattern: pattern, options: .caseInsensitive))? - .firstMatch(in: cleanedName, options: [], range: NSRange(cleanedName.startIndex..., in: cleanedName)) != nil { - DLOG("Found system match by pattern '\(pattern)' for system: \(system.name)") - return system - } - } - } - - return nil - } - - /// Matches a system based on the file name - private func matchSystemByFileName(_ fileName: String) async -> PVSystem? { - let systems = PVEmulatorConfiguration.systems - let lowercasedFileName = fileName.lowercased() - let fileExtension = (fileName as NSString).pathExtension.lowercased() - - // First, try to match based on file extension - if let systemsForExtension = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { - if systemsForExtension.count == 1 { - return systemsForExtension[0] - } else if systemsForExtension.count > 1 { - // If multiple systems match the extension, try to narrow it down - for system in systemsForExtension { - if await doesFileNameMatch(lowercasedFileName, forSystem: system) { - return system - } - } - } - } - - // If extension matching fails, try other methods - for system in systems { - if await doesFileNameMatch(lowercasedFileName, forSystem: system) { - return system - } - } - - // If no match found, try querying the OpenVGDB - do { - if let results = try await openVGDB.searchDatabase(usingFilename: fileName), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? Int, - let system = PVEmulatorConfiguration.system(forDatabaseID: systemID) { - return system - } - } catch { - ELOG("Error querying OpenVGDB for filename: \(error.localizedDescription)") - } - - return nil - } - - /// Checks if a file name matches a given system - private func doesFileNameMatch(_ lowercasedFileName: String, forSystem system: PVSystem) async -> Bool { - // Check if the filename contains the system's name or abbreviation - if lowercasedFileName.contains(system.name.lowercased()) || - lowercasedFileName.contains(system.shortName.lowercased()) { - return true - } - - // Check against known filename patterns for the system - let patterns = filenamePatterns(forSystem: system) - for pattern in patterns { - if lowercasedFileName.range(of: pattern, options: .regularExpression) != nil { - return true - } - } - - // Check against a list of known game titles for the system - if await isKnownGameTitle(lowercasedFileName, forSystem: system) { - return true - } - - return false - } - - /// Checks if a file name matches a known game title for a given system - private func isKnownGameTitle(_ lowercasedFileName: String, forSystem system: PVSystem) async -> Bool { - do { - // Remove file extension and common parenthetical information - let cleanedFileName = cleanFileName(lowercasedFileName) - - // Search the database using the cleaned filename - if let results = try await openVGDB.searchDatabase(usingFilename: cleanedFileName, systemID: system.openvgDatabaseID) { - // Check if we have any results - if !results.isEmpty { - // Optionally, you can add more strict matching here - for result in results { - if let gameTitle = result["gameTitle"] as? String, - cleanFileName(gameTitle.lowercased()) == cleanedFileName { - return true - } - } - } - } - } catch { - ELOG("Error searching OpenVGDB for known game title: \(error.localizedDescription)") - } - return false - } - - /// Cleans a file name - private func cleanFileName(_ fileName: String) -> String { - var cleaned = fileName.lowercased() - - // Remove file extension - if let dotIndex = cleaned.lastIndex(of: ".") { - cleaned = String(cleaned[.. [String] { - let systemName = system.name.lowercased() - let shortName = system.shortName.lowercased() - - var patterns: [String] = [] - - // Add pattern for full system name - patterns.append("\\b\(systemName)\\b") - - // Add pattern for short name - patterns.append("\\b\(shortName)\\b") - // Add some common variations and abbreviations - switch system.identifier { - case "com.provenance.nes": - patterns.append("\\b(nes|nintendo)\\b") - case "com.provenance.snes": - patterns.append("\\b(snes|super\\s*nintendo)\\b") - case "com.provenance.genesis": - patterns.append("\\b(genesis|mega\\s*drive|md)\\b") - case "com.provenance.gba": - patterns.append("\\b(gba|game\\s*boy\\s*advance)\\b") - case "com.provenance.n64": - patterns.append("\\b(n64|nintendo\\s*64)\\b") - case "com.provenance.psx": - patterns.append("\\b(psx|playstation|ps1)\\b") - case "com.provenance.ps2": - patterns.append("\\b(ps2|playstation\\s*2)\\b") - case "com.provenance.gb": - patterns.append("\\b(gb|game\\s*boy)\\b") - case "com.provenance.3DO": - patterns.append("\\b(3do|panasonic\\s*3do)\\b") - case "com.provenance.3ds": - patterns.append("\\b(3ds|nintendo\\s*3ds)\\b") - case "com.provenance.2600": - patterns.append("\\b(2600|atari\\s*2600|vcs)\\b") - case "com.provenance.5200": - patterns.append("\\b(5200|atari\\s*5200)\\b") - case "com.provenance.7800": - patterns.append("\\b(7800|atari\\s*7800)\\b") - case "com.provenance.jaguar": - patterns.append("\\b(jaguar|atari\\s*jaguar)\\b") - case "com.provenance.colecovision": - patterns.append("\\b(coleco|colecovision)\\b") - case "com.provenance.dreamcast": - patterns.append("\\b(dc|dreamcast|sega\\s*dreamcast)\\b") - case "com.provenance.ds": - patterns.append("\\b(nds|nintendo\\s*ds)\\b") - case "com.provenance.gamegear": - patterns.append("\\b(gg|game\\s*gear|sega\\s*game\\s*gear)\\b") - case "com.provenance.gbc": - patterns.append("\\b(gbc|game\\s*boy\\s*color)\\b") - case "com.provenance.lynx": - patterns.append("\\b(lynx|atari\\s*lynx)\\b") - case "com.provenance.mastersystem": - patterns.append("\\b(sms|master\\s*system|sega\\s*master\\s*system)\\b") - case "com.provenance.neogeo": - patterns.append("\\b(neo\\s*geo|neogeo|neo-geo)\\b") - case "com.provenance.ngp": - patterns.append("\\b(ngp|neo\\s*geo\\s*pocket)\\b") - case "com.provenance.ngpc": - patterns.append("\\b(ngpc|neo\\s*geo\\s*pocket\\s*color)\\b") - case "com.provenance.psp": - patterns.append("\\b(psp|playstation\\s*portable)\\b") - case "com.provenance.saturn": - patterns.append("\\b(saturn|sega\\s*saturn)\\b") - case "com.provenance.32X": - patterns.append("\\b(32x|sega\\s*32x)\\b") - case "com.provenance.segacd": - patterns.append("\\b(scd|sega\\s*cd|mega\\s*cd)\\b") - case "com.provenance.sg1000": - patterns.append("\\b(sg1000|sg-1000|sega\\s*1000)\\b") - case "com.provenance.vb": - patterns.append("\\b(vb|virtual\\s*boy)\\b") - case "com.provenance.ws": - patterns.append("\\b(ws|wonderswan)\\b") - case "com.provenance.wsc": - patterns.append("\\b(wsc|wonderswan\\s*color)\\b") - default: - // For systems without specific patterns, we'll just use the general ones created above - break - } - - return patterns - } - - /// Determines the system for a given candidate file - private func determineSystemFromContent(for candidate: ImportCandidateFile, possibleSystems: [PVSystem]) throws -> PVSystem { - // Implement logic to determine system based on file content or metadata - // This could involve checking file headers, parsing content, or using a database of known games - - let fileName = candidate.filePath.deletingPathExtension().lastPathComponent - - for system in possibleSystems { - do { - if let results = try openVGDB.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), - !results.isEmpty { - ILOG("System determined by filename match in OpenVGDB: \(system.name)") - return system - } - } catch { - ELOG("Error searching OpenVGDB for system \(system.name): \(error.localizedDescription)") - } - } - - // If we couldn't determine the system, try a more detailed search - if let fileMD5 = candidate.md5?.uppercased(), !fileMD5.isEmpty { - do { - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: fileMD5), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? Int, - let system = possibleSystems.first(where: { $0.openvgDatabaseID == systemID }) { - ILOG("System determined by MD5 match in OpenVGDB: \(system.name)") - return system - } - } catch { - ELOG("Error searching OpenVGDB by MD5: \(error.localizedDescription)") - } - } - - // If still no match, try to determine based on file content - // This is a placeholder for more advanced content-based detection - // You might want to implement system-specific logic here - for system in possibleSystems { - if doesFileContentMatch(candidate, forSystem: system) { - ILOG("System determined by file content match: \(system.name)") - return system - } - } - - // If we still couldn't determine the system, return the first possible system as a fallback - WLOG("Could not determine system from content, using first possible system as fallback") - return possibleSystems[0] - } - - /// Checks if a file content matches a given system - private func doesFileContentMatch(_ candidate: ImportCandidateFile, forSystem system: PVSystem) -> Bool { - // Implement system-specific file content matching logic here - // This could involve checking file headers, file structure, or other system-specific traits - // For now, we'll return false as a placeholder - return false - } - - /// Determines the system for a given candidate file - private func determineSystem(for candidate: ImportCandidateFile) async throws -> PVSystem { - guard let md5 = candidate.md5?.uppercased() else { - throw GameImporterError.couldNotCalculateMD5 - } - - let fileExtension = candidate.filePath.pathExtension.lowercased() - - DLOG("Checking MD5: \(md5) for possible BIOS match") - // First check if this is a BIOS file by MD5 - let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) - if !biosMatches.isEmpty { - DLOG("Found BIOS matches: \(biosMatches.map { $0.expectedFilename }.joined(separator: ", "))") - // Copy BIOS to all matching system directories - for bios in biosMatches { - if let system = bios.system { - DLOG("Copying BIOS to system: \(system.name)") - let biosPath = PVEmulatorConfiguration.biosPath(forSystemIdentifier: system.identifier) - .appendingPathComponent(bios.expectedFilename) - try FileManager.default.copyItem(at: candidate.filePath, to: biosPath) - } - } - // Return the first system that uses this BIOS - if let firstSystem = biosMatches.first?.system { - DLOG("Using first matching system for BIOS: \(firstSystem.name)") - return firstSystem - } - } - - // Check if it's a CD-based game first - if PVEmulatorConfiguration.supportedCDFileExtensions.contains(fileExtension) { - if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { - if systems.count == 1 { - return systems[0] - } else if systems.count > 1 { - // For CD games with multiple possible systems, use content detection - return try determineSystemFromContent(for: candidate, possibleSystems: systems) - } - } - } - - // Try to find system by MD5 using OpenVGDB - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? NSNumber { - - // Get all matching systems - let matchingSystems = results.compactMap { result -> PVSystem? in - guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } - return PVEmulatorConfiguration.system(forIdentifier: sysID) - } - - if !matchingSystems.isEmpty { - // Sort by release year and take the oldest - if let oldestSystem = matchingSystems.sorted(by: { $0.releaseYear < $1.releaseYear }).first { - DLOG("System determined by MD5 match (oldest): \(oldestSystem.name) (\(oldestSystem.releaseYear))") - return oldestSystem - } - } - - // Fallback to original single system match if sorting fails - if let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { - DLOG("System determined by MD5 match (fallback): \(system.name)") - return system - } - } - - DLOG("MD5 lookup failed, trying filename matching") - - // Try filename matching next - let fileName = candidate.filePath.lastPathComponent - - if let matchedSystem = await matchSystemByFileName(fileName) { - DLOG("Found system by filename match: \(matchedSystem.name)") - return matchedSystem - } - - let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: candidate.filePath.pathExtension.lowercased()) ?? [] - - // If MD5 lookup fails, try to determine the system based on file extension - if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { - if systems.count == 1 { - return systems[0] - } else if systems.count > 1 { - // If multiple systems support this extension, try to determine based on file content or metadata - return try await determineSystemFromContent(for: candidate, possibleSystems: systems) - } - } - - throw GameImporterError.noSystemMatched - } - - /// Retrieves the system ID from the cache for a given ROM candidate - public func systemIdFromCache(forROMCandidate rom: ImportCandidateFile) -> String? { - guard let md5 = rom.md5 else { - ELOG("MD5 was blank") - return nil - } - if let result = RomDatabase.artMD5DBCache[md5] ?? RomDatabase.getArtCacheByFileName(rom.filePath.lastPathComponent), - let databaseID = result["systemID"] as? Int, - let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { - return systemID - } - return nil - } - - /// Matches a system based on the ROM candidate - public func systemId(forROMCandidate rom: ImportCandidateFile) -> String? { - guard let md5 = rom.md5 else { - ELOG("MD5 was blank") - return nil - } - - let fileName: String = rom.filePath.lastPathComponent - - do { - if let databaseID = try openVGDB.system(forRomMD5: md5, or: fileName), - let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { - return systemID - } else { - ILOG("Could't match \(rom.filePath.lastPathComponent) based off of MD5 {\(md5)}") - return nil - } - } catch { - DLOG("Unable to find rom by MD5: \(error.localizedDescription)") - return nil - } - } -} - -/// ROM Query -public extension GameImporter { - - /// Retrieves ROM information for files at given paths - func getRomInfoForFiles(atPaths paths: [URL], userChosenSystem chosenSystem: System? = nil) async { - // If directory, map out sub directories if folder - let paths: [URL] = paths.compactMap { (url) -> [URL]? in - if url.hasDirectoryPath { - return try? FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) - } else { - return [url] - } - }.joined().map { $0 } - - let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) - await sortedPaths.asyncForEach { path in - do { - try await self._handlePath(path: path, userChosenSystem: chosenSystem) - } catch { - ELOG("\(error)") - } - } // for each - } -} - -// Crap, bad crap -extension GameImporter { - - /// Calculates the MD5 hash for a given game - @objc - public func calculateMD5(forGame game: PVGame) -> String? { - var offset: UInt64 = 0 - - if game.systemIdentifier == SystemIdentifier.SNES.rawValue { - offset = SystemIdentifier.SNES.offset - } else if let system = SystemIdentifier(rawValue: game.systemIdentifier) { - offset = system.offset - } - - let romPath = romsPath.appendingPathComponent(game.romPath, isDirectory: false) - let fm = FileManager.default - if !fm.fileExists(atPath: romPath.path) { - ELOG("Cannot find file at path: \(romPath)") - return nil - } - - return fm.md5ForFile(atPath: romPath.path, fromOffset: offset) - } - - /// Saves the relative path for a given game - func saveRelativePath(_ existingGame: PVGame, partialPath:String, file:URL) { - Task { - if await RomDatabase.gamesCache[partialPath] == nil { - await RomDatabase.addRelativeFileCache(file, game:existingGame) - } - } - } - - /// Handles the import of a path - func _handlePath(path: URL, userChosenSystem chosenSystem: System?) async throws { - // Skip hidden files and directories - if path.lastPathComponent.hasPrefix(".") { - VLOG("Skipping hidden file or directory: \(path.lastPathComponent)") - return - } - - let isDirectory = path.hasDirectoryPath - let filename = path.lastPathComponent - let fileExtensionLower = path.pathExtension.lowercased() - - // Handle directories - if isDirectory { - try await handleDirectory(path: path, chosenSystem: chosenSystem) - return - } - - // Handle files - let systems = try determineSystems(for: path, chosenSystem: chosenSystem) - - // Handle conflicts - if systems.count > 1 { - try await handleSystemConflict(path: path, systems: systems) - return - } - - guard let system = systems.first else { - ELOG("No system matched extension {\(fileExtensionLower)}") - try moveToConflictsDirectory(path: path) - return - } - - try importGame(path: path, system: system) - } - - // Helper functions - - /// Handles a directory - private func handleDirectory(path: URL, chosenSystem: System?) async throws { - guard chosenSystem == nil else { return } - - do { - let subContents = try FileManager.default.contentsOfDirectory(at: path, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) - if subContents.isEmpty { - try await FileManager.default.removeItem(at: path) - ILOG("Deleted empty import folder \(path.path)") - } else { - ILOG("Found non-empty folder in imports dir. Will iterate subcontents for import") - for subFile in subContents { - try await self._handlePath(path: subFile, userChosenSystem: nil) - } - } - } catch { - ELOG("Error handling directory: \(error)") - throw error - } - } - private func determineSystemByMD5(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - guard let md5 = candidate.md5?.uppercased() else { - throw GameImporterError.couldNotCalculateMD5 - } - - DLOG("Attempting MD5 lookup for: \(md5)") - - // Try to find system by MD5 using OpenVGDB - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? NSNumber, - let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { - DLOG("System determined by MD5 match: \(system.name)") - return system - } - - DLOG("No system found by MD5") - return nil - } - - /// Determines the systems for a given path - private func determineSystems(for path: URL, chosenSystem: System?) throws -> [PVSystem] { - if let chosenSystem = chosenSystem { - if let system = RomDatabase.systemCache[chosenSystem.identifier] { - return [system] - } - } - - let fileExtensionLower = path.pathExtension.lowercased() - return PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtensionLower) ?? [] - } - - /// Handles a system conflict - private func handleSystemConflict(path: URL, systems: [PVSystem]) async throws { - let candidate = ImportCandidateFile(filePath: path) - DLOG("Handling system conflict for path: \(path.lastPathComponent)") - DLOG("Possible systems: \(systems.map { $0.name }.joined(separator: ", "))") - - // Try to determine system using all available methods - if let system = try? await determineSystem(for: candidate) { - if systems.contains(system) { - DLOG("Found matching system: \(system.name)") - try importGame(path: path, system: system) - return - } else { - DLOG("Determined system \(system.name) not in possible systems list") - } - } else { - DLOG("Could not determine system automatically") - } - - // Fall back to multiple system handling - DLOG("Falling back to multiple system handling") - try handleMultipleSystemMatch(path: path, systems: systems) - } - - /// Handles a multiple system match - private func handleMultipleSystemMatch(path: URL, systems: [PVSystem]) throws { - let filename = path.lastPathComponent - guard let existingGames = GameImporter.findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: filename) else { - self.encounteredConflicts = true - try moveToConflictsDirectory(path: path) - return - } - - if existingGames.count == 1 { - try importGame(path: path, system: systems.first!) - } else { - self.encounteredConflicts = true - try moveToConflictsDirectory(path: path) - let matchedSystems = systems.map { $0.identifier }.joined(separator: ", ") - let matchedGames = existingGames.map { $0.romPath }.joined(separator: ", ") - WLOG("Scanned game matched with multiple systems {\(matchedSystems)} and multiple existing games \(matchedGames) so we moved \(filename) to conflicts dir. You figure it out!") - } - } - - private func importGame(path: URL, system: PVSystem) throws { - DLOG("Attempting to import game: \(path.lastPathComponent) for system: \(system.name)") - let filename = path.lastPathComponent - let partialPath = (system.identifier as NSString).appendingPathComponent(filename) - let similarName = RomDatabase.altName(path, systemIdentifier: system.identifier) - - DLOG("Checking game cache for partialPath: \(partialPath) or similarName: \(similarName)") - let gamesCache = RomDatabase.gamesCache - - if let existingGame = gamesCache[partialPath] ?? gamesCache[similarName], - system.identifier == existingGame.systemIdentifier { - DLOG("Found existing game in cache, saving relative path") - saveRelativePath(existingGame, partialPath: partialPath, file: path) - } else { - DLOG("No existing game found, starting import to database") - Task.detached(priority: .utility) { - try await self.importToDatabaseROM(atPath: path, system: system, relatedFiles: nil) - } - } - } - - /// Moves a file to the conflicts directory - private func moveToConflictsDirectory(path: URL) throws { - let destination = conflictPath.appendingPathComponent(path.lastPathComponent) - try moveAndOverWrite(sourcePath: path, destinationPath: destination) - } - - /// Imports a ROM to the database - private func importToDatabaseROM(atPath path: URL, system: PVSystem, relatedFiles: [URL]?) async throws { - DLOG("Starting database ROM import for: \(path.lastPathComponent)") - let filename = path.lastPathComponent - let filenameSansExtension = path.deletingPathExtension().lastPathComponent - let title: String = PVEmulatorConfiguration.stripDiscNames(fromFilename: filenameSansExtension) - let destinationDir = (system.identifier as NSString) - let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) - - DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") - let file = PVFile(withURL: path) - let game = PVGame(withFile: file, system: system) - game.romPath = partialPath - game.title = title - game.requiresSync = true - var relatedPVFiles = [PVFile]() - let files = RomDatabase.getFileSystemROMCache(for: system).keys - let name = RomDatabase.altName(path, systemIdentifier: system.identifier) - - DLOG("Searching for related files with name: \(name)") - - await files.asyncForEach { url in - let relativeName = RomDatabase.altName(url, systemIdentifier: system.identifier) - DLOG("Checking file \(url.lastPathComponent) with relative name: \(relativeName)") - if relativeName == name { - DLOG("Found matching related file: \(url.lastPathComponent)") - relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) - } - } - - if let relatedFiles = relatedFiles { - DLOG("Processing \(relatedFiles.count) additional related files") - for url in relatedFiles { - DLOG("Adding related file: \(url.lastPathComponent)") - relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) - } - } - - guard let md5 = calculateMD5(forGame: game)?.uppercased() else { - ELOG("Couldn't calculate MD5 for game \(partialPath)") - throw GameImporterError.couldNotCalculateMD5 - } - DLOG("Calculated MD5: \(md5)") - - // Register import with coordinator - guard await importCoordinator.checkAndRegisterImport(md5: md5) else { - DLOG("Import already in progress for MD5: \(md5)") - throw GameImporterError.romAlreadyExistsInDatabase - } - DLOG("Registered import with coordinator for MD5: \(md5)") - - defer { - Task { - await importCoordinator.completeImport(md5: md5) - DLOG("Completed import coordination for MD5: \(md5)") - } - } - - game.relatedFiles.append(objectsIn: relatedPVFiles) - game.md5Hash = md5 - try await finishUpdateOrImport(ofGame: game, path: path) - } - - /// Finishes the update or import of a game - private func finishUpdateOrImport(ofGame game: PVGame, path: URL) async throws { - // Only process if rom doensn't exist in DB - if await RomDatabase.gamesCache[game.romPath] != nil { - throw GameImporterError.romAlreadyExistsInDatabase - } - var modified = false - var game:PVGame = game - if game.requiresSync { - if importStartedHandler != nil { - let fullpath = PVEmulatorConfiguration.path(forGame: game) - Task { @MainActor in - self.importStartedHandler?(fullpath.path) - } - } - game = lookupInfo(for: game, overwrite: true) - modified = true - } - let wasModified = modified - if finishedImportHandler != nil { - let md5: String = game.md5Hash - // Task { @MainActor in - self.finishedImportHandler?(md5, wasModified) - // } - } - if game.originalArtworkFile == nil { - game = await getArtwork(forGame: game) - } - self.saveGame(game) - } - - /// Saves a game to the database - func saveGame(_ game:PVGame) { - do { - let database = RomDatabase.sharedInstance - try database.writeTransaction { - database.realm.create(PVGame.self, value:game, update:.modified) - } - RomDatabase.addGamesCache(game) - } catch { - ELOG("Couldn't add new game \(error.localizedDescription)") - } - } - - /// Finds any current game that could belong to any of the given systems - fileprivate class func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? { - // Check if existing ROM - - let allGames = RomDatabase.gamesCache.values.filter ({ - $0.romPath.lowercased() == romFilename.lowercased() - }) - /* - let database = RomDatabase.sharedInstance - - let predicate = NSPredicate(format: "romPath CONTAINS[c] %@", PVEmulatorConfiguration.stripDiscNames(fromFilename: romFilename)) - let allGames = database.all(PVGame.self, filter: predicate) - */ - // Optionally filter to specfici systems - if let systems = systems { - //let filteredGames = allGames.filter { systems.contains($0.system) } - var sysIds:[String:Bool]=[:] - systems.forEach({ sysIds[$0.identifier] = true }) - let filteredGames = allGames.filter { sysIds[$0.systemIdentifier] != nil } - return filteredGames.isEmpty ? nil : Array(filteredGames) - } else { - return allGames.isEmpty ? nil : Array(allGames) - } - } - private func handleM3UFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - guard candidate.filePath.pathExtension.lowercased() == "m3u" else { - return nil - } - - DLOG("Handling M3U file: \(candidate.filePath.lastPathComponent)") - - // First try to match the M3U file itself by MD5 - if let system = try? await determineSystem(for: candidate) { - DLOG("Found system match for M3U by MD5: \(system.name)") - return system - } - - // Read M3U contents - let contents = try String(contentsOf: candidate.filePath, encoding: .utf8) - let files = contents.components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty && !$0.hasPrefix("#") } - - DLOG("Found \(files.count) entries in M3U") - - // Try to match first valid file in M3U - for file in files { - let filePath = candidate.filePath.deletingLastPathComponent().appendingPathComponent(file) - guard FileManager.default.fileExists(atPath: filePath.path) else { continue } - - let candidateFile = ImportCandidateFile(filePath: filePath) - if let system = try? await determineSystem(for: candidateFile) { - DLOG("Found system match from M3U entry: \(file) -> \(system.name)") - return system - } - } - - DLOG("No system match found for M3U or its contents") - return nil - } - - private func handleRegularROM(_ candidate: ImportCandidateFile) async throws -> (PVSystem, Bool) { - DLOG("Handling regular ROM file: \(candidate.filePath.lastPathComponent)") - - // 1. Try MD5 match first - if let md5 = candidate.md5?.uppercased() { - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), - !results.isEmpty { - let matchingSystems = results.compactMap { result -> PVSystem? in - guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } - return PVEmulatorConfiguration.system(forIdentifier: sysID) - } - - if matchingSystems.count == 1 { - DLOG("Found single system match by MD5: \(matchingSystems[0].name)") - return (matchingSystems[0], false) - } else if matchingSystems.count > 1 { - DLOG("Found multiple system matches by MD5, moving to conflicts") - return (matchingSystems[0], true) // Return first with conflict flag - } - } - } - - let fileName = candidate.filePath.lastPathComponent - let fileExtension = candidate.filePath.pathExtension.lowercased() - let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: fileExtension) ?? [] - - // 2. Try exact filename match - if let system = await matchSystemByFileName(fileName) { - DLOG("Found system match by exact filename: \(system.name)") - return (system, false) - } - - // 3. Try extension match - if possibleSystems.count == 1 { - DLOG("Single system match by extension: \(possibleSystems[0].name)") - return (possibleSystems[0], false) - } else if possibleSystems.count > 1 { - DLOG("Multiple systems match extension, trying partial name match") - - // 4. Try partial filename system identifier match - if let system = matchSystemByPartialName(fileName, possibleSystems: possibleSystems) { - DLOG("Found system match by partial name: \(system.name)") - return (system, false) - } - - DLOG("No definitive system match, moving to conflicts") - return (possibleSystems[0], true) - } - - throw GameImporterError.systemNotDetermined - } -} diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index c5d4bed30a..34a63be506 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -19,12 +19,29 @@ class ImportViewModel: ObservableObject { public let gameImporter = GameImporter.shared } +func iconNameForFileType(_ type: FileType) -> String { + + switch type { + case .bios: + return "bios_filled" + case .artwork: + return "prov_snes_icon" + case .game: + return "prov_snes_icon" + case .cdRom: + return "prov_ps1_icon" + case .unknown: + return "questionMark" + } +} + // Individual Import Task Row View struct ImportTaskRowView: View { - let item: ImportItem + let item: ImportQueueItem var body: some View { HStack { + //TODO: add icon for fileType VStack(alignment: .leading) { Text(item.url.lastPathComponent) .font(.headline) diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift index df10e5f983..8f1bf68ace 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift @@ -239,12 +239,7 @@ extension PVRootViewController: ImportStatusDelegate { } public func addImportsAction() { -// self.showImportOptionsAlert() - - let importItem = ImportItem(url: URL(fileURLWithPath: "game.rom"), - fileType: .game, - system: "NES") - gameImporter.addImport(importItem) + self.showImportOptionsAlert() } } #endif diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift index fb078e45da..c05d52e731 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift @@ -253,11 +253,11 @@ public final class PVGameLibraryUpdatesController: ObservableObject { Task { let initialScan = await scanInitialFiles(at: importPath) if !initialScan.isEmpty { - await gameImporter.startImport(forPaths: initialScan) + gameImporter.addImports(forPaths: initialScan) } for await extractedFiles in directoryWatcher.extractedFilesStream(at: importPath) { - await gameImporter.startImport(forPaths: extractedFiles) + gameImporter.addImports(forPaths: extractedFiles) } } } @@ -378,14 +378,14 @@ public final class PVGameLibraryUpdatesController: ObservableObject { // Process priority files first if !priorityFiles.isEmpty { DLOG("Starting import for priority files") - await gameImporter.startImport(forPaths: priorityFiles) + gameImporter.addImports(forPaths: priorityFiles) DLOG("Finished importing priority files") } // Then process other files if !otherFiles.isEmpty { DLOG("Starting import for other files") - await gameImporter.startImport(forPaths: otherFiles) + gameImporter.addImports(forPaths: otherFiles) DLOG("Finished importing other files") } } From 9011703b1ed622daa69d89de81e32d24287bf230 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Wed, 6 Nov 2024 15:11:20 -0500 Subject: [PATCH 03/30] huge WIP refactor of GameImporter 1. Introducing a robust ImportQueueItem to track an import and follow it through the system 2. Refactored code to extract behaviors out of GameImporter so it's not a god class 3. Pre compute as much as possible for an ImportQueueItem, such as which systems are valid. 4. Try to simplify the general logic to 3 steps - compute, move, and import into the DB Remaining work 1. Tons of code clean up 2. Proper handling of conflicts 3. Proper handling of CDROMs (need to sort to handle M3U -> Cue -> Bin) 4. Proper handling of directories 5. Unit tests where applicable 6. Ensure validity of imports that flow through this new system. 7. UI for Conflict handling 8. Validate that archives work properly. 9. Validate and fix Artwork --- .../Database/Realm Database/RomDatabase.swift | 3 +- .../Importer/Models/GameImporterError.swift | 3 + .../Importer/Models/ImportCandidateFile.swift | 36 -- .../Importer/Models/ImportQueueItem.swift | 54 +- .../GameImporter/GameImporter+Conflicts.swift | 276 +++++----- .../GameImporter/GameImporter+Files.swift | 488 +++++++----------- .../GameImporter/GameImporter+Importing.swift | 481 ++++++++--------- .../GameImporter/GameImporter+ROMLookup.swift | 304 ----------- .../GameImporter/GameImporter+Roms.swift | 223 ++++---- .../GameImporter/GameImporter+Systems.swift | 92 ++-- .../GameImporter/GameImporter+Utils.swift | 51 +- .../Services/GameImporter/GameImporter.swift | 109 ++-- .../GameImporterDatabaseService.swift | 488 ++++++++++++++++++ .../GameImporterFileService.swift | 207 ++++++++ .../Sources/Systems/SystemIdentifier.swift | 1 + .../PVLibraryTests/GameImporterTests.swift | 32 ++ .../Tests/PVLibraryTests/PVLibraryTests.swift | 2 +- 17 files changed, 1592 insertions(+), 1258 deletions(-) delete mode 100644 PVLibrary/Sources/PVLibrary/Importer/Models/ImportCandidateFile.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift create mode 100644 PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index 4fd582fa49..2b6baea88d 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -572,7 +572,8 @@ public extension RomDatabase { game.title = title if game.releaseID == nil || game.releaseID!.isEmpty { ILOG("Game isn't already matched, going to try to re-match after a rename") - GameImporter.shared.lookupInfo(for: game, overwrite: false) + //TODO: figure out when this happens and fix + //GameImporter.shared.lookupInfo(for: game, overwrite: false) } } } catch { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift index 486a0512d5..2f61d56a8b 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift @@ -13,4 +13,7 @@ public enum GameImporterError: Error, Sendable { case systemNotDetermined case failedToMoveCDROM(Error) case failedToMoveROM(Error) + case unsupportedFile + case noBIOSMatchForBIOSFileType + case unsupportedCDROMFile } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportCandidateFile.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportCandidateFile.swift deleted file mode 100644 index 42bfc44751..0000000000 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportCandidateFile.swift +++ /dev/null @@ -1,36 +0,0 @@ -// -// ImportCandidateFile.swift -// PVLibrary -// -// Created by Joseph Mattiello on 7/23/18. -// Copyright © 2018 Provenance Emu. All rights reserved. -// - -import Foundation - -public struct ImportCandidateFile: Codable { - public let filePath: URL - public var md5: String? { - if let cached = cache.md5 { - return cached - } else { - let computed = FileManager.default.md5ForFile(atPath: filePath.path, fromOffset: 0) - cache.md5 = computed - return computed - } - } - - // TODO: Add CRC and SHA-1 - public init(filePath: URL) { - self.filePath = filePath - } - - // Store a cache in a nested class. - // The struct only contains a reference to the class, not the class itself, - // so the struct cannot prevent the class from mutating. - private final class Cache: Codable { - var md5: String? - } - - private var cache = Cache() -} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift index 68d1e11e92..12d1cce64e 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift @@ -6,6 +6,7 @@ // import SwiftUI +import PVPrimitives // Enum to define the possible statuses of each import public enum ImportStatus: String { @@ -53,14 +54,61 @@ public class ImportQueueItem: Identifiable, ObservableObject { public let id = UUID() public let url: URL public var fileType: FileType - public var system: String // Can be set to the specific system type + public var systems: [PVSystem] // Can be set to the specific system type + public var userChosenSystem: System? + public var destinationUrl: URL? // Observable status for individual imports public var status: ImportStatus = .queued - public init(url: URL, fileType: FileType = .unknown, system: String = "") { + public init(url: URL, fileType: FileType = .unknown) { self.url = url self.fileType = fileType - self.system = system + self.systems = [] + self.userChosenSystem = nil } + + public var md5: String? { + if let cached = cache.md5 { + return cached + } else { + let computed = FileManager.default.md5ForFile(atPath: url.path, fromOffset: 0) + cache.md5 = computed + return computed + } + } + + // Store a cache in a nested class. + // The struct only contains a reference to the class, not the class itself, + // so the struct cannot prevent the class from mutating. + private final class Cache: Codable { + var md5: String? + } + + public func targetSystem() -> PVSystem? { + guard !systems.isEmpty else { + return nil + } + + if (systems.count == 1) { + return systems.first! + } + + if let chosenSystem = userChosenSystem { + + var target:PVSystem? = nil + + for system in systems { + if (chosenSystem.identifier == system.identifier) { + target = system + } + } + + return target + } + + return nil + } + + private var cache = Cache() } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift index c707eabc27..493ed9ccb9 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Conflicts.swift @@ -23,145 +23,139 @@ import SwiftUI extension GameImporter { /// Resolves conflicts with given solutions - public func resolveConflicts(withSolutions solutions: [URL: System]) async { - let importOperation = BlockOperation() - - await solutions.asyncForEach { filePath, system in - let subfolder = system.romsDirectory - - if !FileManager.default.fileExists(atPath: subfolder.path, isDirectory: nil) { - ILOG("Path <\(subfolder.path)> doesn't exist. Creating.") - do { - try FileManager.default.createDirectory(at: subfolder, withIntermediateDirectories: true, attributes: nil) - } catch { - ELOG("Error making conflicts dir <\(subfolder.path)>") - assertionFailure("Error making conflicts dir <\(subfolder.path)>") - } - } - - let sourceFilename: String = filePath.lastPathComponent - let sourcePath: URL = filePath - let destinationPath: URL = subfolder.appendingPathComponent(sourceFilename, isDirectory: false) - - do { - try moveAndOverWrite(sourcePath: sourcePath, destinationPath: destinationPath) - } catch { - ELOG("\(error)") - } - - let relatedFileName: String = sourcePath.deletingPathExtension().lastPathComponent - - let conflictsDirContents = try? FileManager.default.contentsOfDirectory(at: conflictPath, includingPropertiesForKeys: nil, options: []) - conflictsDirContents?.forEach { file in - var fileWithoutExtension: String = file.deletingPathExtension().lastPathComponent - fileWithoutExtension = PVEmulatorConfiguration.stripDiscNames(fromFilename: fileWithoutExtension) - let relatedFileName = PVEmulatorConfiguration.stripDiscNames(fromFilename: relatedFileName) - - if fileWithoutExtension == relatedFileName { - let isCueSheet = destinationPath.pathExtension == "cue" - - if isCueSheet { - let cueSheetPath = destinationPath - if var cuesheetContents = try? String(contentsOf: cueSheetPath, encoding: .utf8) { - let range = (cuesheetContents as NSString).range(of: file.lastPathComponent, options: .caseInsensitive) - - if range.location != NSNotFound { - if let subRange = Range(range, in: cuesheetContents) { - cuesheetContents.replaceSubrange(subRange, with: file.lastPathComponent) - } - - do { - try cuesheetContents.write(to: cueSheetPath, atomically: true, encoding: .utf8) - } catch { - ELOG("Unable to rewrite cuesheet \(destinationPath.path) because \(error.localizedDescription)") - } - } else { - DLOG("Range of string <\(file)> not found in file <\(cueSheetPath.lastPathComponent)>") - } - } - } - - do { - let newDestinationPath = subfolder.appendingPathComponent(file.lastPathComponent, isDirectory: false) - try moveAndOverWrite(sourcePath: file, destinationPath: newDestinationPath) - NSLog("Moving \(file.lastPathComponent) to \(newDestinationPath)") - } catch { - ELOG("Unable to move related file from \(filePath.path) to \(subfolder.path) because: \(error.localizedDescription)") - } - } - } - - importOperation.addExecutionBlock { - Task { - ILOG("Import Files at \(destinationPath)") - if let system = RomDatabase.systemCache[system.identifier] { - RomDatabase.addFileSystemROMCache(system) - } - await self.getRomInfoForFiles(atPaths: [destinationPath], userChosenSystem: system) - } - } - } - - let completionOperation = BlockOperation { - if self.completionHandler != nil { - DispatchQueue.main.async(execute: { () -> Void in - self.completionHandler?(false) - }) - } - } - - completionOperation.addDependency(importOperation) - serialImportQueue.addOperation(importOperation) - serialImportQueue.addOperation(completionOperation) - } - - /// Handles a system conflict - internal func handleSystemConflict(path: URL, systems: [PVSystem]) async throws { - let candidate = ImportCandidateFile(filePath: path) - DLOG("Handling system conflict for path: \(path.lastPathComponent)") - DLOG("Possible systems: \(systems.map { $0.name }.joined(separator: ", "))") - - // Try to determine system using all available methods - if let system = try? await determineSystem(for: candidate) { - if systems.contains(system) { - DLOG("Found matching system: \(system.name)") - try importGame(path: path, system: system) - return - } else { - DLOG("Determined system \(system.name) not in possible systems list") - } - } else { - DLOG("Could not determine system automatically") - } - - // Fall back to multiple system handling - DLOG("Falling back to multiple system handling") - try handleMultipleSystemMatch(path: path, systems: systems) - } - - /// Handles a multiple system match - internal func handleMultipleSystemMatch(path: URL, systems: [PVSystem]) throws { - let filename = path.lastPathComponent - guard let existingGames = GameImporter.findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: filename) else { - self.encounteredConflicts = true - try moveToConflictsDirectory(path: path) - return - } - - if existingGames.count == 1 { - try importGame(path: path, system: systems.first!) - } else { - self.encounteredConflicts = true - try moveToConflictsDirectory(path: path) - let matchedSystems = systems.map { $0.identifier }.joined(separator: ", ") - let matchedGames = existingGames.map { $0.romPath }.joined(separator: ", ") - WLOG("Scanned game matched with multiple systems {\(matchedSystems)} and multiple existing games \(matchedGames) so we moved \(filename) to conflicts dir. You figure it out!") - } - } - - /// Moves a file to the conflicts directory - internal func moveToConflictsDirectory(path: URL) throws { - let destination = conflictPath.appendingPathComponent(path.lastPathComponent) - try moveAndOverWrite(sourcePath: path, destinationPath: destination) - } +// public func resolveConflicts(withSolutions solutions: [URL: System]) async { +// let importOperation = BlockOperation() +// +// await solutions.asyncForEach { filePath, system in +// let subfolder = system.romsDirectory +// +// if !FileManager.default.fileExists(atPath: subfolder.path, isDirectory: nil) { +// ILOG("Path <\(subfolder.path)> doesn't exist. Creating.") +// do { +// try FileManager.default.createDirectory(at: subfolder, withIntermediateDirectories: true, attributes: nil) +// } catch { +// ELOG("Error making conflicts dir <\(subfolder.path)>") +// assertionFailure("Error making conflicts dir <\(subfolder.path)>") +// } +// } +// +// let sourceFilename: String = filePath.lastPathComponent +// let sourcePath: URL = filePath +// let destinationPath: URL = subfolder.appendingPathComponent(sourceFilename, isDirectory: false) +// +// do { +// try moveAndOverWrite(sourcePath: sourcePath, destinationPath: destinationPath) +// } catch { +// ELOG("\(error)") +// } +// +// let relatedFileName: String = sourcePath.deletingPathExtension().lastPathComponent +// +// let conflictsDirContents = try? FileManager.default.contentsOfDirectory(at: conflictPath, includingPropertiesForKeys: nil, options: []) +// conflictsDirContents?.forEach { file in +// var fileWithoutExtension: String = file.deletingPathExtension().lastPathComponent +// fileWithoutExtension = PVEmulatorConfiguration.stripDiscNames(fromFilename: fileWithoutExtension) +// let relatedFileName = PVEmulatorConfiguration.stripDiscNames(fromFilename: relatedFileName) +// +// if fileWithoutExtension == relatedFileName { +// let isCueSheet = destinationPath.pathExtension == "cue" +// +// if isCueSheet { +// let cueSheetPath = destinationPath +// if var cuesheetContents = try? String(contentsOf: cueSheetPath, encoding: .utf8) { +// let range = (cuesheetContents as NSString).range(of: file.lastPathComponent, options: .caseInsensitive) +// +// if range.location != NSNotFound { +// if let subRange = Range(range, in: cuesheetContents) { +// cuesheetContents.replaceSubrange(subRange, with: file.lastPathComponent) +// } +// +// do { +// try cuesheetContents.write(to: cueSheetPath, atomically: true, encoding: .utf8) +// } catch { +// ELOG("Unable to rewrite cuesheet \(destinationPath.path) because \(error.localizedDescription)") +// } +// } else { +// DLOG("Range of string <\(file)> not found in file <\(cueSheetPath.lastPathComponent)>") +// } +// } +// } +// +// do { +// let newDestinationPath = subfolder.appendingPathComponent(file.lastPathComponent, isDirectory: false) +// try moveAndOverWrite(sourcePath: file, destinationPath: newDestinationPath) +// NSLog("Moving \(file.lastPathComponent) to \(newDestinationPath)") +// } catch { +// ELOG("Unable to move related file from \(filePath.path) to \(subfolder.path) because: \(error.localizedDescription)") +// } +// } +// } +// +// importOperation.addExecutionBlock { +// Task { +// ILOG("Import Files at \(destinationPath)") +// if let system = RomDatabase.systemCache[system.identifier] { +// RomDatabase.addFileSystemROMCache(system) +// } +// await self.getRomInfoForFiles(atPaths: [destinationPath], userChosenSystem: system) +// } +// } +// } +// +// let completionOperation = BlockOperation { +// if self.completionHandler != nil { +// DispatchQueue.main.async(execute: { () -> Void in +// self.completionHandler?(false) +// }) +// } +// } +// +// completionOperation.addDependency(importOperation) +// serialImportQueue.addOperation(importOperation) +// serialImportQueue.addOperation(completionOperation) +// } +// +// /// Handles a system conflict +// internal func handleSystemConflict(path: URL, systems: [PVSystem]) async throws { +// let candidate = ImportCandidateFile(filePath: path) +// DLOG("Handling system conflict for path: \(path.lastPathComponent)") +// DLOG("Possible systems: \(systems.map { $0.name }.joined(separator: ", "))") +// +// // Try to determine system using all available methods +// if let system = try? await determineSystem(for: candidate) { +// if systems.contains(system) { +// DLOG("Found matching system: \(system.name)") +// try importGame(path: path, system: system) +// return +// } else { +// DLOG("Determined system \(system.name) not in possible systems list") +// } +// } else { +// DLOG("Could not determine system automatically") +// } +// +// // Fall back to multiple system handling +// DLOG("Falling back to multiple system handling") +// try handleMultipleSystemMatch(path: path, systems: systems) +// } +// +// /// Handles a multiple system match +// internal func handleMultipleSystemMatch(path: URL, systems: [PVSystem]) throws { +// let filename = path.lastPathComponent +// guard let existingGames = GameImporter.findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: filename) else { +// self.encounteredConflicts = true +// try moveToConflictsDirectory(path: path) +// return +// } +// +// if existingGames.count == 1 { +// try importGame(path: path, system: systems.first!) +// } else { +// self.encounteredConflicts = true +// try moveToConflictsDirectory(path: path) +// let matchedSystems = systems.map { $0.identifier }.joined(separator: ", ") +// let matchedGames = existingGames.map { $0.romPath }.joined(separator: ", ") +// WLOG("Scanned game matched with multiple systems {\(matchedSystems)} and multiple existing games \(matchedGames) so we moved \(filename) to conflicts dir. You figure it out!") +// } +// } } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift index e38141e303..8ff7a2c889 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Files.swift @@ -10,295 +10,203 @@ import PVSupport import RealmSwift extension GameImporter { - /// Moves a CD-ROM to the appropriate subfolder - internal func moveCDROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> [URL]? { - guard isCDROM(candidate.filePath) else { - return nil - } - - let fileManager = FileManager.default - let fileName = candidate.filePath.lastPathComponent - - guard let system = try? await determineSystem(for: candidate) else { - throw GameImporterError.unsupportedSystem - } - - let destinationFolder = system.romsDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - - do { - try fileManager.createDirectory(at: destinationFolder, withIntermediateDirectories: true, attributes: nil) - try fileManager.moveItem(at: candidate.filePath, to: destinationPath) - let relatedFiles = try await moveRelatedFiles(for: candidate, to: destinationFolder) - return [destinationPath] + relatedFiles - } catch { - throw GameImporterError.failedToMoveCDROM(error) - } - } - /// Moves a ROM to the appropriate subfolder - internal func moveROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> URL? { - guard !isCDROM(candidate.filePath) else { - return nil - } - - let fileManager = FileManager.default - let fileName = candidate.filePath.lastPathComponent - - // Check first if known BIOS - if let system = try await handleBIOSFile(candidate) { - DLOG("Moving BIOS file to system: \(system.name)") - let destinationFolder = system.biosDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - return try await moveROMFile(candidate, to: destinationPath) - } - - // Check for M3U - if let system = try await handleM3UFile(candidate) { - DLOG("Moving M3U and referenced files to system: \(system.name)") - // Move M3U and all referenced files to system directory - let destinationDir = system.romsDirectory - return try await moveM3UAndReferencedFiles(candidate, to: destinationDir) - } - - // CD-ROM handling - if let system = try await handleCDROMFile(candidate) { - DLOG("Moving CD-ROM files to system: \(system.name)") - let destinationDir = system.romsDirectory - return try await moveCDROMFiles(candidate, to: destinationDir) - } - - // Regular ROM handling - let (system, hasConflict) = try await handleRegularROM(candidate) - let destinationDir = hasConflict ? self.conflictPath : system.romsDirectory - - DLOG("Moving ROM to \(hasConflict ? "conflicts" : "system") directory: \(system.name)") - return try await moveROMFile(candidate, to: destinationDir) - } - - private func handleBIOSFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - guard let md5 = candidate.md5?.uppercased() else { - return nil - } - - // Get all BIOS entries that match this MD5 - let matchingBIOSEntries = PVEmulatorConfiguration.biosEntries.filter { biosEntry in - let frozenBiosEntry = biosEntry.isFrozen ? biosEntry : biosEntry.freeze() - return frozenBiosEntry.expectedMD5.uppercased() == md5 - } - - if !matchingBIOSEntries.isEmpty { - // Get the first matching system - if let firstBIOSEntry = matchingBIOSEntries.first { - let frozenBiosEntry = firstBIOSEntry.isFrozen ? firstBIOSEntry : firstBIOSEntry.freeze() - - // Move file to BIOS directory - let destinationURL = frozenBiosEntry.expectedPath - try await moveROMFile(candidate, to: destinationURL) - - // Update BIOS entry in Realm - try await MainActor.run { - let realm = try Realm() - try realm.write { - if let thawedBios = frozenBiosEntry.thaw() { - let biosFile = PVFile(withURL: destinationURL) - thawedBios.file = biosFile - } - } - } - - return frozenBiosEntry.system - } - } - - return nil - } - - private func handleCDROMFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - let `extension` = candidate.filePath.pathExtension.lowercased() - guard PVEmulatorConfiguration.supportedCDFileExtensions.contains(`extension`) else { - return nil - } - - DLOG("Handling CD-ROM file: \(candidate.filePath.lastPathComponent)") - - // First try MD5 matching - if let system = try? await determineSystem(for: candidate) { - DLOG("Found system match for CD-ROM by MD5: \(system.name)") - return system - } - - // If cue file, try to match its bin file - if `extension` == "cue" { - if let binFile = try findAssociatedBinFile(for: candidate) { - DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") - let binCandidate = ImportCandidateFile(filePath: binFile) - if let system = try? await determineSystem(for: binCandidate) { - DLOG("Found system match from associated bin file: \(system.name)") - return system - } - } - } - - // Try exact filename match - if let system = await matchSystemByFileName(candidate.filePath.lastPathComponent) { - DLOG("Found system match by filename: \(system.name)") - return system - } - - DLOG("No system match found for CD-ROM file") - return nil - } - - /// Move a `ImportCandidateFile` to a destination, creating the destination directory if needed - private func moveROMFile(_ romFile: ImportCandidateFile, to destination: URL) async throws -> URL { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let destPath = destination.appendingPathComponent(romFile.filePath.lastPathComponent) - try FileManager.default.moveItem(at: romFile.filePath, to: destPath) - DLOG("Moved ROM file to: \(destPath.path)") - return destPath - } - - private func findAssociatedBinFile(for cueFile: ImportCandidateFile) throws -> URL? { - let cueContents = try String(contentsOf: cueFile.filePath, encoding: .utf8) - let lines = cueContents.components(separatedBy: .newlines) - - // Look for FILE "something.bin" BINARY line - for line in lines { - let components = line.trimmingCharacters(in: .whitespaces) - .components(separatedBy: "\"") - guard components.count >= 2, - line.lowercased().contains("file") && line.lowercased().contains("binary") else { - continue - } - - let binFileName = components[1] - let binPath = cueFile.filePath.deletingLastPathComponent().appendingPathComponent(binFileName) - - if FileManager.default.fileExists(atPath: binPath.path) { - return binPath - } - } - - return nil - } - - private func moveM3UAndReferencedFiles(_ m3uFile: ImportCandidateFile, to destination: URL) async throws -> URL { - let contents = try String(contentsOf: m3uFile.filePath, encoding: .utf8) - let files = contents.components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty && !$0.hasPrefix("#") } - - // Create destination directory if needed - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - - // Move all referenced files - for file in files { - let sourcePath = m3uFile.filePath.deletingLastPathComponent().appendingPathComponent(file) - let destPath = destination.appendingPathComponent(file) - - if FileManager.default.fileExists(atPath: sourcePath.path) { - try FileManager.default.moveItem(at: sourcePath, to: destPath) - DLOG("Moved M3U referenced file: \(file)") - } - } - - // Move M3U file itself - let m3uDestPath = destination.appendingPathComponent(m3uFile.filePath.lastPathComponent) - try FileManager.default.moveItem(at: m3uFile.filePath, to: m3uDestPath) - DLOG("Moved M3U file to: \(m3uDestPath.path)") - - return m3uDestPath - } - - private func moveCDROMFiles(_ cdFile: ImportCandidateFile, to destination: URL) async throws -> URL { - let fileManager = FileManager.default - try fileManager.createDirectory(at: destination, withIntermediateDirectories: true) - - let `extension` = cdFile.filePath.pathExtension.lowercased() - let destPath = destination.appendingPathComponent(cdFile.filePath.lastPathComponent) - - // If it's a cue file, move both cue and bin - if `extension` == "cue" { - if let binPath = try findAssociatedBinFile(for: cdFile) { - let binDestPath = destination.appendingPathComponent(binPath.lastPathComponent) - try fileManager.moveItem(at: binPath, to: binDestPath) - DLOG("Moved bin file to: \(binDestPath.path)") - } - } - - // Move the main CD-ROM file - try fileManager.moveItem(at: cdFile.filePath, to: destPath) - DLOG("Moved CD-ROM file to: \(destPath.path)") - - return destPath - } - - /// Moves related files for a given candidate - private func moveRelatedFiles(for candidate: ImportCandidateFile, to destinationFolder: URL) async throws -> [URL] { - let fileManager = FileManager.default - let fileName = candidate.filePath.deletingPathExtension().lastPathComponent - let sourceFolder = candidate.filePath.deletingLastPathComponent() - - let relatedFiles = try fileManager.contentsOfDirectory(at: sourceFolder, includingPropertiesForKeys: nil) - .filter { $0.deletingPathExtension().lastPathComponent == fileName && $0 != candidate.filePath } - - return try await withThrowingTaskGroup(of: URL.self) { group in - for file in relatedFiles { - group.addTask { - let destination = destinationFolder.appendingPathComponent(file.lastPathComponent) - try fileManager.moveItem(at: file, to: destination) - return destination - } - } - - var movedFiles: [URL] = [] - for try await movedFile in group { - movedFiles.append(movedFile) - } - return movedFiles - } - } - - /// Moves a file and overwrites if it already exists at the destination - public func moveAndOverWrite(sourcePath: URL, destinationPath: URL) throws { - let fileManager = FileManager.default - - // If file exists at destination, remove it first - if fileManager.fileExists(atPath: destinationPath.path) { - try fileManager.removeItem(at: destinationPath) - } - - // Now move the file - try fileManager.moveItem(at: sourcePath, to: destinationPath) - } - - /// BIOS entry matching - private func biosEntryMatching(candidateFile: ImportCandidateFile) -> [PVBIOS]? { - let fileName = candidateFile.filePath.lastPathComponent - var matchingBioses = Set() - - DLOG("Checking if file is BIOS: \(fileName)") - - // First try to match by filename - if let biosEntry = PVEmulatorConfiguration.biosEntry(forFilename: fileName) { - DLOG("Found BIOS match by filename: \(biosEntry.expectedFilename)") - matchingBioses.insert(biosEntry) - } - - // Then try to match by MD5 - if let md5 = candidateFile.md5?.uppercased(), - let md5Entry = PVEmulatorConfiguration.biosEntry(forMD5: md5) { - DLOG("Found BIOS match by MD5: \(md5Entry.expectedFilename)") - matchingBioses.insert(md5Entry) - } - - if !matchingBioses.isEmpty { - let matches = Array(matchingBioses) - DLOG("Found \(matches.count) BIOS matches") - return matches - } - - return nil - } +// /// Moves a CD-ROM to the appropriate subfolder +// internal func moveCDROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> [URL]? { +// guard isCDROM(candidate.filePath) else { +// return nil +// } +// +// let fileManager = FileManager.default +// let fileName = candidate.filePath.lastPathComponent +// +// guard let system = try? await determineSystem(for: candidate) else { +// throw GameImporterError.unsupportedSystem +// } +// +// //TODO: we might want a better way of handling cue/bin files... +// // If cue file, try to match its bin file +// if `extension` == "cue" { +// if let binFile = try findAssociatedBinFile(for: candidate) { +// DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") +// let binCandidate = ImportCandidateFile(filePath: binFile) +// if let system = try? await determineSystem(for: binCandidate) { +// DLOG("Found system match from associated bin file: \(system.name)") +// return system +// } +// } +// } +// +// let destinationFolder = system.romsDirectory +// let destinationPath = destinationFolder.appendingPathComponent(fileName) +// +// do { +// try fileManager.createDirectory(at: destinationFolder, withIntermediateDirectories: true, attributes: nil) +// try fileManager.moveItem(at: candidate.filePath, to: destinationPath) +// let relatedFiles = try await moveRelatedFiles(for: candidate, to: destinationFolder) +// return [destinationPath] + relatedFiles +// } catch { +// throw GameImporterError.failedToMoveCDROM(error) +// } +// } +// +// /// Moves a ROM to the appropriate subfolder +// internal func moveROM(toAppropriateSubfolder candidate: ImportCandidateFile) async throws -> URL? { +// guard !isCDROM(candidate.filePath) else { +// return nil +// } +// +// let fileManager = FileManager.default +// let fileName = candidate.filePath.lastPathComponent +// +// +// // Regular ROM handling +// let (system, hasConflict) = try await handleRegularROM(candidate) +// let destinationDir = hasConflict ? self.conflictPath : system.romsDirectory +// +// DLOG("Moving ROM to \(hasConflict ? "conflicts" : "system") directory: \(system.name)") +// return try await moveROMFile(candidate, to: destinationDir) +// } +// +// private func handleCDROMFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { +// let `extension` = candidate.filePath.pathExtension.lowercased() +// guard PVEmulatorConfiguration.supportedCDFileExtensions.contains(`extension`) else { +// return nil +// } +// +// DLOG("Handling CD-ROM file: \(candidate.filePath.lastPathComponent)") +// +// // First try MD5 matching +// if let system = try? await determineSystem(for: candidate) { +// DLOG("Found system match for CD-ROM by MD5: \(system.name)") +// return system +// } +// +// // If cue file, try to match its bin file +// if `extension` == "cue" { +// if let binFile = try findAssociatedBinFile(for: candidate) { +// DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") +// let binCandidate = ImportCandidateFile(filePath: binFile) +// if let system = try? await determineSystem(for: binCandidate) { +// DLOG("Found system match from associated bin file: \(system.name)") +// return system +// } +// } +// } +// +// // Try exact filename match +// if let system = await matchSystemByFileName(candidate.filePath.lastPathComponent) { +// DLOG("Found system match by filename: \(system.name)") +// return system +// } +// +// DLOG("No system match found for CD-ROM file") +// return nil +// } +// +// private func moveM3UAndReferencedFiles(_ m3uFile: ImportCandidateFile, to destination: URL) async throws -> URL { +// let contents = try String(contentsOf: m3uFile.filePath, encoding: .utf8) +// let files = contents.components(separatedBy: .newlines) +// .map { $0.trimmingCharacters(in: .whitespaces) } +// .filter { !$0.isEmpty && !$0.hasPrefix("#") } +// +// // Create destination directory if needed +// try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) +// +// // Move all referenced files +// for file in files { +// let sourcePath = m3uFile.filePath.deletingLastPathComponent().appendingPathComponent(file) +// let destPath = destination.appendingPathComponent(file) +// +// if FileManager.default.fileExists(atPath: sourcePath.path) { +// try FileManager.default.moveItem(at: sourcePath, to: destPath) +// DLOG("Moved M3U referenced file: \(file)") +// } +// } +// +// // Move M3U file itself +// let m3uDestPath = destination.appendingPathComponent(m3uFile.filePath.lastPathComponent) +// try FileManager.default.moveItem(at: m3uFile.filePath, to: m3uDestPath) +// DLOG("Moved M3U file to: \(m3uDestPath.path)") +// +// return m3uDestPath +// } +// +// private func moveCDROMFiles(_ cdFile: ImportCandidateFile, to destination: URL) async throws -> URL { +// let fileManager = FileManager.default +// try fileManager.createDirectory(at: destination, withIntermediateDirectories: true) +// +// let `extension` = cdFile.filePath.pathExtension.lowercased() +// let destPath = destination.appendingPathComponent(cdFile.filePath.lastPathComponent) +// +// // If it's a cue file, move both cue and bin +// if `extension` == "cue" { +// if let binPath = try findAssociatedBinFile(for: cdFile) { +// let binDestPath = destination.appendingPathComponent(binPath.lastPathComponent) +// try fileManager.moveItem(at: binPath, to: binDestPath) +// DLOG("Moved bin file to: \(binDestPath.path)") +// } +// } +// +// // Move the main CD-ROM file +// try fileManager.moveItem(at: cdFile.filePath, to: destPath) +// DLOG("Moved CD-ROM file to: \(destPath.path)") +// +// return destPath +// } +// +// /// Moves related files for a given candidate +// private func moveRelatedFiles(for candidate: ImportCandidateFile, to destinationFolder: URL) async throws -> [URL] { +// let fileManager = FileManager.default +// let fileName = candidate.filePath.deletingPathExtension().lastPathComponent +// let sourceFolder = candidate.filePath.deletingLastPathComponent() +// +// let relatedFiles = try fileManager.contentsOfDirectory(at: sourceFolder, includingPropertiesForKeys: nil) +// .filter { $0.deletingPathExtension().lastPathComponent == fileName && $0 != candidate.filePath } +// +// return try await withThrowingTaskGroup(of: URL.self) { group in +// for file in relatedFiles { +// group.addTask { +// let destination = destinationFolder.appendingPathComponent(file.lastPathComponent) +// try fileManager.moveItem(at: file, to: destination) +// return destination +// } +// } +// +// var movedFiles: [URL] = [] +// for try await movedFile in group { +// movedFiles.append(movedFile) +// } +// return movedFiles +// } +// } +// +// /// BIOS entry matching +// private func biosEntryMatching(candidateFile: ImportCandidateFile) -> [PVBIOS]? { +// let fileName = candidateFile.filePath.lastPathComponent +// var matchingBioses = Set() +// +// DLOG("Checking if file is BIOS: \(fileName)") +// +// // First try to match by filename +// if let biosEntry = PVEmulatorConfiguration.biosEntry(forFilename: fileName) { +// DLOG("Found BIOS match by filename: \(biosEntry.expectedFilename)") +// matchingBioses.insert(biosEntry) +// } +// +// // Then try to match by MD5 +// if let md5 = candidateFile.md5?.uppercased(), +// let md5Entry = PVEmulatorConfiguration.biosEntry(forMD5: md5) { +// DLOG("Found BIOS match by MD5: \(md5Entry.expectedFilename)") +// matchingBioses.insert(md5Entry) +// } +// +// if !matchingBioses.isEmpty { +// let matches = Array(matchingBioses) +// DLOG("Found \(matches.count) BIOS matches") +// return matches +// } +// +// return nil +// } } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift index 4264904f41..cf9fe08b14 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Importing.swift @@ -24,268 +24,237 @@ import SwiftUI extension GameImporter { /// Imports files from given paths /// //goal is to make this private - public func importFiles(atPaths paths: [URL]) async throws -> [URL] { - let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) - var importedFiles: [URL] = [] - - for path in sortedPaths { - do { - if let importedFile = try await importSingleFile(at: path) { - importedFiles.append(importedFile) - } - } catch { - ELOG("Failed to import file at \(path.path): \(error.localizedDescription)") - } - } - - return importedFiles - } +// public func importFiles(atPaths paths: [URL]) async throws -> [URL] { +// let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) +// var importedFiles: [URL] = [] +// +// for path in sortedPaths { +// do { +// if let importedFile = try await importSingleFile(at: path) { +// importedFiles.append(importedFile) +// } +// } catch { +// ELOG("Failed to import file at \(path.path): \(error.localizedDescription)") +// } +// } +// +// return importedFiles +// } - /// Imports a single file from the given path - internal func importSingleFile(at path: URL) async throws -> URL? { - guard FileManager.default.fileExists(atPath: path.path) else { - WLOG("File doesn't exist at \(path.path)") - return nil - } - - if isCDROM(path) { - return try await handleCDROM(at: path) - } else if isArtwork(path) { - return try await handleArtwork(at: path) - } else { - return try await handleROM(at: path) - } - } +// /// Imports a single file from the given path +// internal func importSingleFile(at path: URL) async throws -> URL? { +// guard FileManager.default.fileExists(atPath: path.path) else { +// WLOG("File doesn't exist at \(path.path)") +// return nil +// } +// +// if isCDROM(path) { +// return try await handleCDROM(at: path) +// } else if isArtwork(path) { +// return try await handleArtwork(at: path) +// } else { +// return try await handleROM(at: path) +// } +// } - /// Handles importing a CD-ROM - internal func handleCDROM(at path: URL) async throws -> URL? { - let movedToPaths = try await moveCDROM(toAppropriateSubfolder: ImportCandidateFile(filePath: path)) - if let movedToPaths = movedToPaths { - let pathsString = movedToPaths.map { $0.path }.joined(separator: ", ") - VLOG("Found a CD. Moved files to the following paths \(pathsString)") - } - return nil - } +// /// Handles importing a CD-ROM +// internal func handleCDROM(at path: URL) async throws -> URL? { +// let movedToPaths = try await moveCDROM(toAppropriateSubfolder: ImportCandidateFile(filePath: path)) +// if let movedToPaths = movedToPaths { +// let pathsString = movedToPaths.map { $0.path }.joined(separator: ", ") +// VLOG("Found a CD. Moved files to the following paths \(pathsString)") +// } +// return nil +// } - /// Handles importing artwork - internal func handleArtwork(at path: URL) async throws -> URL? { - if let game = await GameImporter.importArtwork(fromPath: path) { - ILOG("Found artwork \(path.lastPathComponent) for game <\(game.title)>") - } - return nil - } +// /// Handles importing artwork +// internal func handleArtwork(at path: URL) async throws -> URL? { +// if let game = await GameImporter.importArtwork(fromPath: path) { +// ILOG("Found artwork \(path.lastPathComponent) for game <\(game.title)>") +// } +// return nil +// } - /// Handles importing a ROM - internal func handleROM(at path: URL) async throws -> URL? { - let candidate = ImportCandidateFile(filePath: path) - return try await moveROM(toAppropriateSubfolder: candidate) - } +// /// Handles importing a ROM +// internal func handleROM(at path: URL) async throws -> URL? { +// let candidate = ImportCandidateFile(filePath: path) +// return try await moveROM(toAppropriateSubfolder: candidate) +// } - /// Starts an import for the given paths - internal func startImport(forPaths paths: [URL]) async { - // Pre-sort - let paths = PVEmulatorConfiguration.sortImportURLs(urls: paths) - let scanOperation = BlockOperation { - Task { - do { - let newPaths = try await self.importFiles(atPaths: paths) - await self.getRomInfoForFiles(atPaths: newPaths, userChosenSystem: nil) - } catch { - ELOG("\(error)") - } - } - } - - let completionOperation = BlockOperation { - if self.completionHandler != nil { - DispatchQueue.main.sync(execute: { () -> Void in - self.completionHandler?(self.encounteredConflicts) - }) - } - } - - completionOperation.addDependency(scanOperation) - serialImportQueue.addOperation(scanOperation) - serialImportQueue.addOperation(completionOperation) - } +// /// Starts an import for the given paths +// internal func startImport(forPaths paths: [URL]) async { +// // Pre-sort +// let paths = PVEmulatorConfiguration.sortImportURLs(urls: paths) +// let scanOperation = BlockOperation { +// Task { +// do { +// let newPaths = try await self.importFiles(atPaths: paths) +// await self.getRomInfoForFiles(atPaths: newPaths, userChosenSystem: nil) +// } catch { +// ELOG("\(error)") +// } +// } +// } +// +// let completionOperation = BlockOperation { +// if self.completionHandler != nil { +// DispatchQueue.main.sync(execute: { () -> Void in +// self.completionHandler?(self.encounteredConflicts) +// }) +// } +// } +// +// completionOperation.addDependency(scanOperation) +// serialImportQueue.addOperation(scanOperation) +// serialImportQueue.addOperation(completionOperation) +// } +// +// /// Handles the import of a path +// internal func _handlePath(path: URL, userChosenSystem chosenSystem: System?) async throws { +// // Skip hidden files and directories +// if path.lastPathComponent.hasPrefix(".") { +// VLOG("Skipping hidden file or directory: \(path.lastPathComponent)") +// return +// } +// +// let isDirectory = path.hasDirectoryPath +// let filename = path.lastPathComponent +// let fileExtensionLower = path.pathExtension.lowercased() +// +// // Handle directories +// //TODO: strangle this back to the importer queue somehow... +// if isDirectory { +// try await handleDirectory(path: path, chosenSystem: chosenSystem) +// return +// } +// +// // Handle files +// let systems = try determineSystems(for: path, chosenSystem: chosenSystem) +// +// // Handle conflicts +// if systems.count > 1 { +// try await handleSystemConflict(path: path, systems: systems) +// return +// } +// +// //this is the case where there was no matching system - should this even happne? +// guard let system = systems.first else { +// ELOG("No system matched extension {\(fileExtensionLower)}") +// try moveToConflictsDirectory(path: path) +// return +// } +// +// try importGame(path: path, system: system) +// } - /// Handles the import of a path - internal func _handlePath(path: URL, userChosenSystem chosenSystem: System?) async throws { - // Skip hidden files and directories - if path.lastPathComponent.hasPrefix(".") { - VLOG("Skipping hidden file or directory: \(path.lastPathComponent)") - return - } - - let isDirectory = path.hasDirectoryPath - let filename = path.lastPathComponent - let fileExtensionLower = path.pathExtension.lowercased() - - // Handle directories - //TODO: strangle this back to the importer queue somehow... - if isDirectory { - try await handleDirectory(path: path, chosenSystem: chosenSystem) - return - } - - // Handle files - let systems = try determineSystems(for: path, chosenSystem: chosenSystem) - - // Handle conflicts - if systems.count > 1 { - try await handleSystemConflict(path: path, systems: systems) - return - } - - //this is the case where there was no matching system - should this even happne? - guard let system = systems.first else { - ELOG("No system matched extension {\(fileExtensionLower)}") - try moveToConflictsDirectory(path: path) - return - } - - try importGame(path: path, system: system) - } +// /// Handles a directory +// /// //TODO: strangle this back to the importer queue +// internal func handleDirectory(path: URL, chosenSystem: System?) async throws { +// guard chosenSystem == nil else { return } +// +// do { +// let subContents = try FileManager.default.contentsOfDirectory(at: path, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) +// if subContents.isEmpty { +// try await FileManager.default.removeItem(at: path) +// ILOG("Deleted empty import folder \(path.path)") +// } else { +// ILOG("Found non-empty folder in imports dir. Will iterate subcontents for import") +// for subFile in subContents { +// try await self._handlePath(path: subFile, userChosenSystem: nil) +// } +// } +// } catch { +// ELOG("Error handling directory: \(error)") +// throw error +// } +// } - /// Handles a directory - /// //TODO: strangle this back to the importer queue - internal func handleDirectory(path: URL, chosenSystem: System?) async throws { - guard chosenSystem == nil else { return } - - do { - let subContents = try FileManager.default.contentsOfDirectory(at: path, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) - if subContents.isEmpty { - try await FileManager.default.removeItem(at: path) - ILOG("Deleted empty import folder \(path.path)") - } else { - ILOG("Found non-empty folder in imports dir. Will iterate subcontents for import") - for subFile in subContents { - try await self._handlePath(path: subFile, userChosenSystem: nil) - } - } - } catch { - ELOG("Error handling directory: \(error)") - throw error - } - } +// internal func handleM3UFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { +// guard candidate.filePath.pathExtension.lowercased() == "m3u" else { +// return nil +// } +// +// DLOG("Handling M3U file: \(candidate.filePath.lastPathComponent)") +// +// // First try to match the M3U file itself by MD5 +// if let system = try? await determineSystem(for: candidate) { +// DLOG("Found system match for M3U by MD5: \(system.name)") +// return system +// } +// +// // Read M3U contents +// let contents = try String(contentsOf: candidate.filePath, encoding: .utf8) +// let files = contents.components(separatedBy: .newlines) +// .map { $0.trimmingCharacters(in: .whitespaces) } +// .filter { !$0.isEmpty && !$0.hasPrefix("#") } +// +// DLOG("Found \(files.count) entries in M3U") +// +// // Try to match first valid file in M3U +// for file in files { +// let filePath = candidate.filePath.deletingLastPathComponent().appendingPathComponent(file) +// guard FileManager.default.fileExists(atPath: filePath.path) else { continue } +// +// let candidateFile = ImportCandidateFile(filePath: filePath) +// if let system = try? await determineSystem(for: candidateFile) { +// DLOG("Found system match from M3U entry: \(file) -> \(system.name)") +// return system +// } +// } +// +// DLOG("No system match found for M3U or its contents") +// return nil +// } - internal func handleM3UFile(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - guard candidate.filePath.pathExtension.lowercased() == "m3u" else { - return nil - } - - DLOG("Handling M3U file: \(candidate.filePath.lastPathComponent)") - - // First try to match the M3U file itself by MD5 - if let system = try? await determineSystem(for: candidate) { - DLOG("Found system match for M3U by MD5: \(system.name)") - return system - } - - // Read M3U contents - let contents = try String(contentsOf: candidate.filePath, encoding: .utf8) - let files = contents.components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty && !$0.hasPrefix("#") } - - DLOG("Found \(files.count) entries in M3U") - - // Try to match first valid file in M3U - for file in files { - let filePath = candidate.filePath.deletingLastPathComponent().appendingPathComponent(file) - guard FileManager.default.fileExists(atPath: filePath.path) else { continue } - - let candidateFile = ImportCandidateFile(filePath: filePath) - if let system = try? await determineSystem(for: candidateFile) { - DLOG("Found system match from M3U entry: \(file) -> \(system.name)") - return system - } - } - - DLOG("No system match found for M3U or its contents") - return nil - } - - internal func handleRegularROM(_ candidate: ImportCandidateFile) async throws -> (PVSystem, Bool) { - DLOG("Handling regular ROM file: \(candidate.filePath.lastPathComponent)") - - // 1. Try MD5 match first - if let md5 = candidate.md5?.uppercased() { - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), - !results.isEmpty { - let matchingSystems = results.compactMap { result -> PVSystem? in - guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } - return PVEmulatorConfiguration.system(forIdentifier: sysID) - } - - if matchingSystems.count == 1 { - DLOG("Found single system match by MD5: \(matchingSystems[0].name)") - return (matchingSystems[0], false) - } else if matchingSystems.count > 1 { - DLOG("Found multiple system matches by MD5, moving to conflicts") - return (matchingSystems[0], true) // Return first with conflict flag - } - } - } - - let fileName = candidate.filePath.lastPathComponent - let fileExtension = candidate.filePath.pathExtension.lowercased() - let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: fileExtension) ?? [] - - // 2. Try exact filename match - if let system = await matchSystemByFileName(fileName) { - DLOG("Found system match by exact filename: \(system.name)") - return (system, false) - } - - // 3. Try extension match - if possibleSystems.count == 1 { - DLOG("Single system match by extension: \(possibleSystems[0].name)") - return (possibleSystems[0], false) - } else if possibleSystems.count > 1 { - DLOG("Multiple systems match extension, trying partial name match") - - // 4. Try partial filename system identifier match - if let system = matchSystemByPartialName(fileName, possibleSystems: possibleSystems) { - DLOG("Found system match by partial name: \(system.name)") - return (system, false) - } - - DLOG("No definitive system match, moving to conflicts") - return (possibleSystems[0], true) - } - - throw GameImporterError.systemNotDetermined - } - - /// Finishes the update or import of a game - internal func finishUpdateOrImport(ofGame game: PVGame, path: URL) async throws { - // Only process if rom doensn't exist in DB - if await RomDatabase.gamesCache[game.romPath] != nil { - throw GameImporterError.romAlreadyExistsInDatabase - } - var modified = false - var game:PVGame = game - if game.requiresSync { - if importStartedHandler != nil { - let fullpath = PVEmulatorConfiguration.path(forGame: game) - Task { @MainActor in - self.importStartedHandler?(fullpath.path) - } - } - game = lookupInfo(for: game, overwrite: true) - modified = true - } - let wasModified = modified - if finishedImportHandler != nil { - let md5: String = game.md5Hash - // Task { @MainActor in - self.finishedImportHandler?(md5, wasModified) - // } - } - if game.originalArtworkFile == nil { - game = await getArtwork(forGame: game) - } - self.saveGame(game) - } +// internal func handleRegularROM(_ candidate: ImportCandidateFile) async throws -> (PVSystem, Bool) { +// DLOG("Handling regular ROM file: \(candidate.filePath.lastPathComponent)") +// +// // 1. Try MD5 match first +// if let md5 = candidate.md5?.uppercased() { +// if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), +// !results.isEmpty { +// let matchingSystems = results.compactMap { result -> PVSystem? in +// guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } +// return PVEmulatorConfiguration.system(forIdentifier: sysID) +// } +// +// if matchingSystems.count == 1 { +// DLOG("Found single system match by MD5: \(matchingSystems[0].name)") +// return (matchingSystems[0], false) +// } else if matchingSystems.count > 1 { +// DLOG("Found multiple system matches by MD5, moving to conflicts") +// return (matchingSystems[0], true) // Return first with conflict flag +// } +// } +// } +// +// let fileName = candidate.filePath.lastPathComponent +// let fileExtension = candidate.filePath.pathExtension.lowercased() +// let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: fileExtension) ?? [] +// +// // 2. Try exact filename match +// if let system = await matchSystemByFileName(fileName) { +// DLOG("Found system match by exact filename: \(system.name)") +// return (system, false) +// } +// +// // 3. Try extension match +// if possibleSystems.count == 1 { +// DLOG("Single system match by extension: \(possibleSystems[0].name)") +// return (possibleSystems[0], false) +// } else if possibleSystems.count > 1 { +// DLOG("Multiple systems match extension, trying partial name match") +// +// // 4. Try partial filename system identifier match +// if let system = matchSystemByPartialName(fileName, possibleSystems: possibleSystems) { +// DLOG("Found system match by partial name: \(system.name)") +// return (system, false) +// } +// +// DLOG("No definitive system match, moving to conflicts") +// return (possibleSystems[0], true) +// } +// +// throw GameImporterError.systemNotDetermined +// } } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift index 7bbca59914..85001c5a3d 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift @@ -12,309 +12,5 @@ import Systems // MARK: - ROM Lookup public extension GameImporter { - - @discardableResult - func lookupInfo(for game: PVGame, overwrite: Bool = true) -> PVGame { - game.requiresSync = false - if game.md5Hash.isEmpty { - let offset: UInt64 - switch game.systemIdentifier { - case "com.provenance.nes": - offset = 16 - default: - offset = 0 - } - let romFullPath = romsPath.appendingPathComponent(game.romPath).path - if let md5Hash = FileManager.default.md5ForFile(atPath: romFullPath, fromOffset: offset) { - game.md5Hash = md5Hash - } - } - guard !game.md5Hash.isEmpty else { - NSLog("Game md5 has was empty") - return game - } - var resultsMaybe: [[String: Any]]? - do { - if let result = RomDatabase.getArtCache(game.md5Hash.uppercased(), systemIdentifier:game.systemIdentifier) { - resultsMaybe=[result] - } else { - resultsMaybe = try searchDatabase(usingKey: "romHashMD5", value: game.md5Hash.uppercased(), systemID: game.systemIdentifier) - } - } catch { - ELOG("\(error.localizedDescription)") - } - if resultsMaybe == nil || resultsMaybe!.isEmpty { //PVEmulatorConfiguration.supportedROMFileExtensions.contains(game.file.url.pathExtension.lowercased()) { - let fileName: String = game.file.url.lastPathComponent - // Remove any extraneous stuff in the rom name such as (U), (J), [T+Eng] etc - let nonCharRange: NSRange = (fileName as NSString).rangeOfCharacter(from: GameImporter.charset) - var gameTitleLen: Int - if nonCharRange.length > 0, nonCharRange.location > 1 { - gameTitleLen = nonCharRange.location - 1 - } else { - gameTitleLen = fileName.count - } - let subfileName = String(fileName.prefix(gameTitleLen)) - do { - if let result = RomDatabase.getArtCacheByFileName(subfileName, systemIdentifier:game.systemIdentifier) { - resultsMaybe=[result] - } else { - resultsMaybe = try searchDatabase(usingKey: "romFileName", value: subfileName, systemID: game.systemIdentifier) - } - } catch { - ELOG("\(error.localizedDescription)") - } - } - guard let results = resultsMaybe, !results.isEmpty else { - // the file maybe exists but was wiped from DB, - // try to re-import and rescan if can - // skip re-import during artwork download process - /* - let urls = importFiles(atPaths: [game.url]) - if !urls.isEmpty { - lookupInfo(for: game, overwrite: overwrite) - return - } else { - DLOG("Unable to find ROM \(game.romPath) in DB") - try? database.writeTransaction { - game.requiresSync = false - } - return - } - */ - return game - } - var chosenResultMaybe: [String: Any]? = - // Search by region id - results.first { (dict) -> Bool in - DLOG("region id: \(dict["regionID"] as? Int ?? 0)") - // Region ids USA = 21, Japan = 13 - return (dict["regionID"] as? Int) == 21 - } - ?? // If nothing, search by region string, could be a comma sepearted list - results.first { (dict) -> Bool in - DLOG("region: \(dict["region"] ?? "nil")") - // Region ids USA = 21, Japan = 13 - return (dict["region"] as? String)?.uppercased().contains("USA") ?? false - } - if chosenResultMaybe == nil { - if results.count > 1 { - ILOG("Query returned \(results.count) possible matches. Failed to matcha USA version by string or release ID int. Going to choose the first that exists in the DB.") - } - chosenResultMaybe = results.first - } - //write at the end of fininshOrUpdateImport - //autoreleasepool { - // do { - game.requiresSync = false - guard let chosenResult = chosenResultMaybe else { - NSLog("Unable to find ROM \(game.romPath) in OpenVGDB") - return game - } - /* Optional results - gameTitle - boxImageURL - region - gameDescription - boxBackURL - developer - publisher - year - genres [comma array string] - referenceURL - releaseID - regionID - systemShortName - serial - */ - if let title = chosenResult["gameTitle"] as? String, !title.isEmpty, overwrite || game.title.isEmpty { - // Remove just (Disc 1) from the title. Discs with other numbers will retain their names - let revisedTitle = title.replacingOccurrences(of: "\\ \\(Disc 1\\)", with: "", options: .regularExpression) - game.title = revisedTitle - } - - if let boxImageURL = chosenResult["boxImageURL"] as? String, !boxImageURL.isEmpty, overwrite || game.originalArtworkURL.isEmpty { - game.originalArtworkURL = boxImageURL - } - - if let regionName = chosenResult["region"] as? String, !regionName.isEmpty, overwrite || game.regionName == nil { - game.regionName = regionName - } - - if let regionID = chosenResult["regionID"] as? Int, overwrite || game.regionID.value == nil { - game.regionID.value = regionID - } - - if let gameDescription = chosenResult["gameDescription"] as? String, !gameDescription.isEmpty, overwrite || game.gameDescription == nil { - let options = [NSAttributedString.DocumentReadingOptionKey.documentType: NSAttributedString.DocumentType.html] - if let data = gameDescription.data(using: .isoLatin1) { - do { - let htmlDecodedGameDescription = try NSMutableAttributedString(data: data, options: options, documentAttributes: nil) - game.gameDescription = htmlDecodedGameDescription.string.replacingOccurrences(of: "(\\.|\\!|\\?)([A-Z][A-Za-z\\s]{2,})", with: "$1\n\n$2", options: .regularExpression) - } catch { - ELOG("\(error.localizedDescription)") - } - } - } - - if let boxBackURL = chosenResult["boxBackURL"] as? String, !boxBackURL.isEmpty, overwrite || game.boxBackArtworkURL == nil { - game.boxBackArtworkURL = boxBackURL - } - - if let developer = chosenResult["developer"] as? String, !developer.isEmpty, overwrite || game.developer == nil { - game.developer = developer - } - - if let publisher = chosenResult["publisher"] as? String, !publisher.isEmpty, overwrite || game.publisher == nil { - game.publisher = publisher - } - - if let genres = chosenResult["genres"] as? String, !genres.isEmpty, overwrite || game.genres == nil { - game.genres = genres - } - - if let releaseDate = chosenResult["releaseDate"] as? String, !releaseDate.isEmpty, overwrite || game.publishDate == nil { - game.publishDate = releaseDate - } - - if let referenceURL = chosenResult["referenceURL"] as? String, !referenceURL.isEmpty, overwrite || game.referenceURL == nil { - game.referenceURL = referenceURL - } - - if let releaseID = chosenResult["releaseID"] as? NSNumber, !releaseID.stringValue.isEmpty, overwrite || game.releaseID == nil { - game.releaseID = releaseID.stringValue - } - - if let systemShortName = chosenResult["systemShortName"] as? String, !systemShortName.isEmpty, overwrite || game.systemShortName == nil { - game.systemShortName = systemShortName - } - - if let romSerial = chosenResult["serial"] as? String, !romSerial.isEmpty, overwrite || game.romSerial == nil { - game.romSerial = romSerial - } - // } catch { - // ELOG("Failed to update game \(game.title) : \(error.localizedDescription)") - // } - //} - return game - } - - @discardableResult - func getArtwork(forGame game: PVGame) async -> PVGame { - var url = game.originalArtworkURL - if url.isEmpty { - return game - } - if PVMediaCache.fileExists(forKey: url) { - if let localURL = PVMediaCache.filePath(forKey: url) { - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) - game.originalArtworkFile = file - return game - } - } - DLOG("Starting Artwork download for \(url)") - // Note: Evil hack for bad domain in DB - url = url.replacingOccurrences(of: "gamefaqs1.cbsistatic.com/box/", with: "gamefaqs.gamespot.com/a/box/") - guard let artworkURL = URL(string: url) else { - ELOG("url is invalid url <\(url)>") - return game - } - let request = URLRequest(url: artworkURL) - var imageData:Data? - - if let response = try? await URLSession.shared.data(for: request), (response.1 as? HTTPURLResponse)?.statusCode == 200 { - imageData = response.0 - } -// let semaphore = DispatchSemaphore(value: 0) -// let task = URLSession.shared.dataTask(with: request) { dataMaybe, urlResponseMaybe, error in -// if let error = error { -// ELOG("Network error: \(error.localizedDescription)") -// } else { -// if let urlResponse = urlResponseMaybe as? HTTPURLResponse, -// urlResponse.statusCode == 200 { -// imageData = dataMaybe -// } -// } -// semaphore.signal() -// } -// task.resume() -// _ = semaphore.wait(timeout: .distantFuture) - func artworkCompletion(artworkURL: String) { - if self.finishedArtworkHandler != nil { - DispatchQueue.main.sync(execute: { () -> Void in - ILOG("Calling finishedArtworkHandler \(artworkURL)") - self.finishedArtworkHandler!(artworkURL) - }) - } else { - ELOG("finishedArtworkHandler == nil") - } - } - if let data = imageData { -#if os(macOS) - if let artwork = NSImage(data: data) { - do { - let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) - game.originalArtworkFile = file - } catch { ELOG("\(error.localizedDescription)") } - } -#elseif !os(watchOS) - if let artwork = UIImage(data: data) { - do { - let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) - game.originalArtworkFile = file - } catch { ELOG("\(error.localizedDescription)") } - } -#endif - } - artworkCompletion(artworkURL: url) - return game - } - func releaseID(forCRCs crcs: Set) -> String? { - return openVGDB.releaseID(forCRCs: crcs) - } - - enum DatabaseQueryError: Error { - case invalidSystemID - } - - func searchDatabase(usingKey key: String, value: String, systemID: SystemIdentifier) throws -> [[String: NSObject]]? { - guard let systemIDInt = PVEmulatorConfiguration.databaseID(forSystemID: systemID.rawValue) else { - throw DatabaseQueryError.invalidSystemID - } - - return try openVGDB.searchDatabase(usingKey: key, value: value, systemID: systemIDInt) - } - - func searchDatabase(usingKey key: String, value: String, systemID: String) throws -> [[String: NSObject]]? { - guard let systemIDInt = PVEmulatorConfiguration.databaseID(forSystemID: systemID) else { - throw DatabaseQueryError.invalidSystemID - } - - return try openVGDB.searchDatabase(usingKey: key, value: value, systemID: systemIDInt) - } - - // TODO: This was a quick copy of the general version for filenames specifically - func searchDatabase(usingFilename filename: String, systemID: String) throws -> [[String: NSObject]]? { - guard let systemIDInt = PVEmulatorConfiguration.databaseID(forSystemID: systemID) else { - throw DatabaseQueryError.invalidSystemID - } - - return try openVGDB.searchDatabase(usingFilename: filename, systemID: systemIDInt) - } - func searchDatabase(usingFilename filename: String, systemIDs: [String]) throws -> [[String: NSObject]]? { - let systemIDsInts: [Int] = systemIDs.compactMap { PVEmulatorConfiguration.databaseID(forSystemID: $0) } - guard !systemIDsInts.isEmpty else { - throw DatabaseQueryError.invalidSystemID - } - - return try openVGDB.searchDatabase(usingFilename: filename, systemIDs: systemIDsInts) - } - - static var charset: CharacterSet = { - var c = CharacterSet.punctuationCharacters - c.remove(charactersIn: ",-+&.'") - return c - }() } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift index 134a0d13be..91a4a444ea 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Roms.swift @@ -23,123 +23,110 @@ import SwiftUI public extension GameImporter { - /// Retrieves ROM information for files at given paths - func getRomInfoForFiles(atPaths paths: [URL], userChosenSystem chosenSystem: System? = nil) async { - //TODO: split this off at the importer queue entry point so we can remove here - // If directory, map out sub directories if folder - let paths: [URL] = paths.compactMap { (url) -> [URL]? in - if url.hasDirectoryPath { - return try? FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) - } else { - return [url] - } - }.joined().map { $0 } - - let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) - await sortedPaths.asyncForEach { path in - do { - try await self._handlePath(path: path, userChosenSystem: chosenSystem) - } catch { - //TODO: what do i do here? I probably could just let this throw... - ELOG("\(error)") - } - } // for each - } +// /// Retrieves ROM information for files at given paths +// func getRomInfoForFiles(atPaths paths: [URL], userChosenSystem chosenSystem: System? = nil) async { +// //TODO: split this off at the importer queue entry point so we can remove here +// // If directory, map out sub directories if folder +// let paths: [URL] = paths.compactMap { (url) -> [URL]? in +// if url.hasDirectoryPath { +// return try? FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) +// } else { +// return [url] +// } +// }.joined().map { $0 } +// +// let sortedPaths = PVEmulatorConfiguration.sortImportURLs(urls: paths) +// await sortedPaths.asyncForEach { path in +// do { +// try await self._handlePath(path: path, userChosenSystem: chosenSystem) +// } catch { +// //TODO: what do i do here? I probably could just let this throw... +// ELOG("\(error)") +// } +// } // for each +// } - internal func importGame(path: URL, system: PVSystem) throws { - DLOG("Attempting to import game: \(path.lastPathComponent) for system: \(system.name)") - let filename = path.lastPathComponent - let partialPath = (system.identifier as NSString).appendingPathComponent(filename) - let similarName = RomDatabase.altName(path, systemIdentifier: system.identifier) - - DLOG("Checking game cache for partialPath: \(partialPath) or similarName: \(similarName)") - let gamesCache = RomDatabase.gamesCache - - if let existingGame = gamesCache[partialPath] ?? gamesCache[similarName], - system.identifier == existingGame.systemIdentifier { - DLOG("Found existing game in cache, saving relative path") - saveRelativePath(existingGame, partialPath: partialPath, file: path) - } else { - DLOG("No existing game found, starting import to database") - Task.detached(priority: .utility) { - try await self.importToDatabaseROM(atPath: path, system: system, relatedFiles: nil) - } - } - } - - /// Imports a ROM to the database - internal func importToDatabaseROM(atPath path: URL, system: PVSystem, relatedFiles: [URL]?) async throws { - DLOG("Starting database ROM import for: \(path.lastPathComponent)") - let filename = path.lastPathComponent - let filenameSansExtension = path.deletingPathExtension().lastPathComponent - let title: String = PVEmulatorConfiguration.stripDiscNames(fromFilename: filenameSansExtension) - let destinationDir = (system.identifier as NSString) - let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) - - DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") - let file = PVFile(withURL: path) - let game = PVGame(withFile: file, system: system) - game.romPath = partialPath - game.title = title - game.requiresSync = true - var relatedPVFiles = [PVFile]() - let files = RomDatabase.getFileSystemROMCache(for: system).keys - let name = RomDatabase.altName(path, systemIdentifier: system.identifier) - - DLOG("Searching for related files with name: \(name)") - - await files.asyncForEach { url in - let relativeName = RomDatabase.altName(url, systemIdentifier: system.identifier) - DLOG("Checking file \(url.lastPathComponent) with relative name: \(relativeName)") - if relativeName == name { - DLOG("Found matching related file: \(url.lastPathComponent)") - relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) - } - } - - if let relatedFiles = relatedFiles { - DLOG("Processing \(relatedFiles.count) additional related files") - for url in relatedFiles { - DLOG("Adding related file: \(url.lastPathComponent)") - relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) - } - } - - guard let md5 = calculateMD5(forGame: game)?.uppercased() else { - ELOG("Couldn't calculate MD5 for game \(partialPath)") - throw GameImporterError.couldNotCalculateMD5 - } - DLOG("Calculated MD5: \(md5)") - - // Register import with coordinator - guard await importCoordinator.checkAndRegisterImport(md5: md5) else { - DLOG("Import already in progress for MD5: \(md5)") - throw GameImporterError.romAlreadyExistsInDatabase - } - DLOG("Registered import with coordinator for MD5: \(md5)") - - defer { - Task { - await importCoordinator.completeImport(md5: md5) - DLOG("Completed import coordination for MD5: \(md5)") - } - } - - game.relatedFiles.append(objectsIn: relatedPVFiles) - game.md5Hash = md5 - try await finishUpdateOrImport(ofGame: game, path: path) - } - - /// Saves a game to the database - func saveGame(_ game:PVGame) { - do { - let database = RomDatabase.sharedInstance - try database.writeTransaction { - database.realm.create(PVGame.self, value:game, update:.modified) - } - RomDatabase.addGamesCache(game) - } catch { - ELOG("Couldn't add new game \(error.localizedDescription)") - } - } +// internal func importGame(path: URL, system: PVSystem) throws { +// DLOG("Attempting to import game: \(path.lastPathComponent) for system: \(system.name)") +// let filename = path.lastPathComponent +// let partialPath = (system.identifier as NSString).appendingPathComponent(filename) +// let similarName = RomDatabase.altName(path, systemIdentifier: system.identifier) +// +// DLOG("Checking game cache for partialPath: \(partialPath) or similarName: \(similarName)") +// let gamesCache = RomDatabase.gamesCache +// +// if let existingGame = gamesCache[partialPath] ?? gamesCache[similarName], +// system.identifier == existingGame.systemIdentifier { +// DLOG("Found existing game in cache, saving relative path") +// saveRelativePath(existingGame, partialPath: partialPath, file: path) +// } else { +// DLOG("No existing game found, starting import to database") +// Task.detached(priority: .utility) { +// try await self.importToDatabaseROM(atPath: path, system: system, relatedFiles: nil) +// } +// } +// } +// +// /// Imports a ROM to the database +// internal func importToDatabaseROM(atPath path: URL, system: PVSystem, relatedFiles: [URL]?) async throws { +// DLOG("Starting database ROM import for: \(path.lastPathComponent)") +// let filename = path.lastPathComponent +// let filenameSansExtension = path.deletingPathExtension().lastPathComponent +// let title: String = PVEmulatorConfiguration.stripDiscNames(fromFilename: filenameSansExtension) +// let destinationDir = (system.identifier as NSString) +// let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) +// +// DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") +// let file = PVFile(withURL: path) +// let game = PVGame(withFile: file, system: system) +// game.romPath = partialPath +// game.title = title +// game.requiresSync = true +// var relatedPVFiles = [PVFile]() +// let files = RomDatabase.getFileSystemROMCache(for: system).keys +// let name = RomDatabase.altName(path, systemIdentifier: system.identifier) +// +// DLOG("Searching for related files with name: \(name)") +// +// await files.asyncForEach { url in +// let relativeName = RomDatabase.altName(url, systemIdentifier: system.identifier) +// DLOG("Checking file \(url.lastPathComponent) with relative name: \(relativeName)") +// if relativeName == name { +// DLOG("Found matching related file: \(url.lastPathComponent)") +// relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) +// } +// } +// +// if let relatedFiles = relatedFiles { +// DLOG("Processing \(relatedFiles.count) additional related files") +// for url in relatedFiles { +// DLOG("Adding related file: \(url.lastPathComponent)") +// relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) +// } +// } +// +// guard let md5 = calculateMD5(forGame: game)?.uppercased() else { +// ELOG("Couldn't calculate MD5 for game \(partialPath)") +// throw GameImporterError.couldNotCalculateMD5 +// } +// DLOG("Calculated MD5: \(md5)") +// +// // Register import with coordinator +// guard await importCoordinator.checkAndRegisterImport(md5: md5) else { +// DLOG("Import already in progress for MD5: \(md5)") +// throw GameImporterError.romAlreadyExistsInDatabase +// } +// DLOG("Registered import with coordinator for MD5: \(md5)") +// +// defer { +// Task { +// await importCoordinator.completeImport(md5: md5) +// DLOG("Completed import coordination for MD5: \(md5)") +// } +// } +// +// game.relatedFiles.append(objectsIn: relatedPVFiles) +// game.md5Hash = md5 +// try await finishUpdateOrImport(ofGame: game, path: path) +// } } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift index 2a28ad0bb2..e8b0af4a03 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift @@ -69,8 +69,9 @@ extension GameImporter { } // If no match found, try querying the OpenVGDB + //TODO: fix me do { - if let results = try await openVGDB.searchDatabase(usingFilename: fileName), + if let results = try openVGDB.searchDatabase(usingFilename: fileName), let firstResult = results.first, let systemID = firstResult["systemID"] as? Int, let system = PVEmulatorConfiguration.system(forDatabaseID: systemID) { @@ -114,7 +115,7 @@ extension GameImporter { let cleanedFileName = cleanFileName(lowercasedFileName) // Search the database using the cleaned filename - if let results = try await openVGDB.searchDatabase(usingFilename: cleanedFileName, systemID: system.openvgDatabaseID) { + if let results = try openVGDB.searchDatabase(usingFilename: cleanedFileName, systemID: system.openvgDatabaseID) { // Check if we have any results if !results.isEmpty { // Optionally, you can add more strict matching here @@ -255,11 +256,11 @@ extension GameImporter { } /// Determines the system for a given candidate file - private func determineSystemFromContent(for candidate: ImportCandidateFile, possibleSystems: [PVSystem]) throws -> PVSystem { + private func determineSystemFromContent(for queueItem: ImportQueueItem, possibleSystems: [PVSystem]) throws -> PVSystem { // Implement logic to determine system based on file content or metadata // This could involve checking file headers, parsing content, or using a database of known games - let fileName = candidate.filePath.deletingPathExtension().lastPathComponent + let fileName = queueItem.url.deletingPathExtension().lastPathComponent for system in possibleSystems { do { @@ -274,7 +275,7 @@ extension GameImporter { } // If we couldn't determine the system, try a more detailed search - if let fileMD5 = candidate.md5?.uppercased(), !fileMD5.isEmpty { + if let fileMD5 = queueItem.md5?.uppercased(), !fileMD5.isEmpty { do { if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: fileMD5), let firstResult = results.first, @@ -284,6 +285,7 @@ extension GameImporter { return system } } catch { + //what to do here, since this results in no system? ELOG("Error searching OpenVGDB by MD5: \(error.localizedDescription)") } } @@ -292,19 +294,20 @@ extension GameImporter { // This is a placeholder for more advanced content-based detection // You might want to implement system-specific logic here for system in possibleSystems { - if doesFileContentMatch(candidate, forSystem: system) { + if doesFileContentMatch(queueItem, forSystem: system) { ILOG("System determined by file content match: \(system.name)") return system } } // If we still couldn't determine the system, return the first possible system as a fallback + //TODO: should we actually do this? WLOG("Could not determine system from content, using first possible system as fallback") return possibleSystems[0] } /// Checks if a file content matches a given system - private func doesFileContentMatch(_ candidate: ImportCandidateFile, forSystem system: PVSystem) -> Bool { + private func doesFileContentMatch(_ queueItem: ImportQueueItem, forSystem system: PVSystem) -> Bool { // Implement system-specific file content matching logic here // This could involve checking file headers, file structure, or other system-specific traits // For now, we'll return false as a placeholder @@ -312,42 +315,35 @@ extension GameImporter { } /// Determines the system for a given candidate file - internal func determineSystem(for candidate: ImportCandidateFile) async throws -> PVSystem { - guard let md5 = candidate.md5?.uppercased() else { + internal func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] { + guard let md5 = queueItem.md5?.uppercased() else { throw GameImporterError.couldNotCalculateMD5 } - let fileExtension = candidate.filePath.pathExtension.lowercased() + let fileExtension = queueItem.url.pathExtension.lowercased() DLOG("Checking MD5: \(md5) for possible BIOS match") // First check if this is a BIOS file by MD5 - let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) - if !biosMatches.isEmpty { - DLOG("Found BIOS matches: \(biosMatches.map { $0.expectedFilename }.joined(separator: ", "))") - // Copy BIOS to all matching system directories + + if queueItem.fileType == .bios { + let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) + var biosSystemMatches:[PVSystem] = [] for bios in biosMatches { - if let system = bios.system { - DLOG("Copying BIOS to system: \(system.name)") - let biosPath = PVEmulatorConfiguration.biosPath(forSystemIdentifier: system.identifier) - .appendingPathComponent(bios.expectedFilename) - try FileManager.default.copyItem(at: candidate.filePath, to: biosPath) - } - } - // Return the first system that uses this BIOS - if let firstSystem = biosMatches.first?.system { - DLOG("Using first matching system for BIOS: \(firstSystem.name)") - return firstSystem + biosSystemMatches.append(bios.system) } + return biosSystemMatches } - // Check if it's a CD-based game first if PVEmulatorConfiguration.supportedCDFileExtensions.contains(fileExtension) { if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { if systems.count == 1 { - return systems[0] + return [systems[0]] } else if systems.count > 1 { // For CD games with multiple possible systems, use content detection - return try determineSystemFromContent(for: candidate, possibleSystems: systems) + // but allow for the fact that this might not exclude a system + //TODO: fixme + let aSystem = try determineSystemFromContent(for: queueItem, possibleSystems: systems) + return [aSystem] } } } @@ -365,38 +361,40 @@ extension GameImporter { if !matchingSystems.isEmpty { // Sort by release year and take the oldest + //TODO: consider whether this is a good idea? if let oldestSystem = matchingSystems.sorted(by: { $0.releaseYear < $1.releaseYear }).first { + //TODO: is this the right move, i'm not sure - might be better to consider a conflict here DLOG("System determined by MD5 match (oldest): \(oldestSystem.name) (\(oldestSystem.releaseYear))") - return oldestSystem + return [oldestSystem] } } // Fallback to original single system match if sorting fails if let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { DLOG("System determined by MD5 match (fallback): \(system.name)") - return system + return [system] } } DLOG("MD5 lookup failed, trying filename matching") // Try filename matching next - let fileName = candidate.filePath.lastPathComponent + let fileName = queueItem.url.lastPathComponent if let matchedSystem = await matchSystemByFileName(fileName) { DLOG("Found system by filename match: \(matchedSystem.name)") - return matchedSystem + return [matchedSystem] } - let possibleSystems = PVEmulatorConfiguration.systems(forFileExtension: candidate.filePath.pathExtension.lowercased()) ?? [] - // If MD5 lookup fails, try to determine the system based on file extension if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { if systems.count == 1 { - return systems[0] + return systems } else if systems.count > 1 { // If multiple systems support this extension, try to determine based on file content or metadata - return try await determineSystemFromContent(for: candidate, possibleSystems: systems) + //TODO: fixme + let aSystem = try determineSystemFromContent(for: queueItem, possibleSystems: systems) + return [aSystem] } } @@ -404,13 +402,15 @@ extension GameImporter { } /// Retrieves the system ID from the cache for a given ROM candidate - public func systemIdFromCache(forROMCandidate rom: ImportCandidateFile) -> String? { - guard let md5 = rom.md5 else { + public func systemIdFromCache(forQueueItem queueItem: ImportQueueItem) -> String? { + guard let md5 = queueItem.md5 else { ELOG("MD5 was blank") return nil } - if let result = RomDatabase.artMD5DBCache[md5] ?? RomDatabase.getArtCacheByFileName(rom.filePath.lastPathComponent), - let databaseID = result["systemID"] as? Int, + let result = RomDatabase.artMD5DBCache[md5] ?? RomDatabase.getArtCacheByFileName(queueItem.url.lastPathComponent) + + if let _res = result, + let databaseID = _res["systemID"] as? Int, let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { return systemID } @@ -418,20 +418,20 @@ extension GameImporter { } /// Matches a system based on the ROM candidate - public func systemId(forROMCandidate rom: ImportCandidateFile) -> String? { - guard let md5 = rom.md5 else { + public func systemId(forQueueItem queueItem: ImportQueueItem) -> String? { + guard let md5 = queueItem.md5 else { ELOG("MD5 was blank") return nil } - let fileName: String = rom.filePath.lastPathComponent + let fileName: String = queueItem.url.lastPathComponent do { if let databaseID = try openVGDB.system(forRomMD5: md5, or: fileName), let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { return systemID } else { - ILOG("Could't match \(rom.filePath.lastPathComponent) based off of MD5 {\(md5)}") + ILOG("Could't match \(queueItem.url.lastPathComponent) based off of MD5 {\(md5)}") return nil } } catch { @@ -440,8 +440,8 @@ extension GameImporter { } } - internal func determineSystemByMD5(_ candidate: ImportCandidateFile) async throws -> PVSystem? { - guard let md5 = candidate.md5?.uppercased() else { + internal func determineSystemByMD5(_ queueItem: ImportQueueItem) async throws -> PVSystem? { + guard let md5 = queueItem.md5?.uppercased() else { throw GameImporterError.couldNotCalculateMD5 } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift index cca3c1b05e..c967ff2662 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Utils.swift @@ -8,39 +8,10 @@ extension GameImporter { - /// Calculates the MD5 hash for a given game - @objc - public func calculateMD5(forGame game: PVGame) -> String? { - var offset: UInt64 = 0 - - if game.systemIdentifier == SystemIdentifier.SNES.rawValue { - offset = SystemIdentifier.SNES.offset - } else if let system = SystemIdentifier(rawValue: game.systemIdentifier) { - offset = system.offset - } - - let romPath = romsPath.appendingPathComponent(game.romPath, isDirectory: false) - let fm = FileManager.default - if !fm.fileExists(atPath: romPath.path) { - ELOG("Cannot find file at path: \(romPath)") - return nil - } - - return fm.md5ForFile(atPath: romPath.path, fromOffset: offset) - } - - /// Saves the relative path for a given game - func saveRelativePath(_ existingGame: PVGame, partialPath:String, file:URL) { - Task { - if await RomDatabase.gamesCache[partialPath] == nil { - await RomDatabase.addRelativeFileCache(file, game:existingGame) - } - } - } /// Checks if a given ROM file is a CD-ROM - internal func isCDROM(_ romFile: ImportCandidateFile) -> Bool { - return isCDROM(romFile.filePath) + internal func isCDROM(_ queueItem: ImportQueueItem) -> Bool { + return isCDROM(queueItem.url) } /// Checks if a given path is a CD-ROM @@ -51,9 +22,23 @@ extension GameImporter { } /// Checks if a given path is artwork - package func isArtwork(_ path: URL) -> Bool { + internal func isArtwork(_ queueItem: ImportQueueItem) -> Bool { let artworkExtensions = Extensions.artworkExtensions - let fileExtension = path.pathExtension.lowercased() + let fileExtension = queueItem.url.pathExtension.lowercased() return artworkExtensions.contains(fileExtension) } + + internal func isBIOS(_ queueItem: ImportQueueItem) throws -> Bool { + guard let md5 = queueItem.md5?.uppercased() else { + throw GameImporterError.couldNotCalculateMD5 + } + + DLOG("Checking MD5: \(md5) for possible BIOS match") + + // First check if this is a BIOS file by MD5 + let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) + + //it's a bios if it's md5 matches known BIOS + return !biosMatches.isEmpty + } } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 4f3b55e074..5655dbcf18 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -102,9 +102,19 @@ public typealias GameImporterFinishedImportingGameHandler = (_ md5Hash: String, /// Type alias for a closure that handles the finish of getting artwork public typealias GameImporterFinishedGettingArtworkHandler = (_ artworkURL: String?) -> Void - - - +public protocol GameImporting { + func initSystems() async + + var importStatus: String { get } + + var importQueue: [ImportQueueItem] { get } + + var processingState: ProcessingState { get } + + func addImport(_ item: ImportQueueItem) + func addImports(forPaths paths: [URL]) + func startProcessing() +} #if !os(tvOS) @@ -112,7 +122,7 @@ public typealias GameImporterFinishedGettingArtworkHandler = (_ artworkURL: Stri #else @Perceptible #endif -public final class GameImporter: ObservableObject { +public final class GameImporter: GameImporting, ObservableObject { /// Closure called when import starts public var importStartedHandler: GameImporterImportStartedHandler? /// Closure called when import completes @@ -131,7 +141,9 @@ public final class GameImporter: ObservableObject { public var spotlightFinishedImportHandler: GameImporterFinishedImportingGameHandler? /// Singleton instance of GameImporter - public static let shared: GameImporter = GameImporter() + public static let shared: GameImporter = GameImporter(FileManager.default, + GameImporterFileService(), + GameImporterDatabaseService()) /// Instance of OpenVGDB for database operations var openVGDB = OpenVGDB.init() @@ -165,6 +177,9 @@ public final class GameImporter: ObservableObject { public var processingState: ProcessingState = .idle // Observable state for processing status + internal var gameImporterFileService:GameImporterFileServicing + internal var gameImporterDatabaseService:GameImporterDatabaseServicing + // MARK: - Paths /// Path to the documents directory @@ -194,9 +209,19 @@ public final class GameImporter: ObservableObject { internal let importCoordinator = ImportCoordinator() /// Initializes the GameImporter - fileprivate init() { - let fm = FileManager.default + internal init(_ fm: FileManager, + _ fileService:GameImporterFileServicing, + _ databaseService:GameImporterDatabaseServicing) { + gameImporterFileService = fileService + gameImporterDatabaseService = databaseService + + //create defaults createDefaultDirectories(fm: fm) + + //set the romsPath propery of the db service, since it needs access + gameImporterDatabaseService.setRomsPath(url: romsPath) + gameImporterDatabaseService.setOpenVGDB(openVGDB) + } /// Creates default directories @@ -300,7 +325,7 @@ public final class GameImporter: ObservableObject { public func addImports(forPaths paths: [URL]) { paths.forEach({ (url) in - addImportItemToQueue(ImportQueueItem(url: url, fileType: .unknown, system: "")) + addImportItemToQueue(ImportQueueItem(url: url, fileType: .unknown)) }) startProcessing() @@ -336,13 +361,13 @@ public final class GameImporter: ObservableObject { private func processItem(_ item: ImportQueueItem) async { ILOG("GameImportQueue - processing item in queue: \(item.url)") item.status = .processing - updateImporterStatus("Importing \(item.url.lastPathComponent) for \(item.system)") + updateImporterStatus("Importing \(item.url.lastPathComponent)") do { // Simulate file processing try await performImport(for: item) item.status = .success - updateImporterStatus("Completed \(item.url.lastPathComponent) for \(item.system)") + updateImporterStatus("Completed \(item.url.lastPathComponent)") ILOG("GameImportQueue - processing item in queue: \(item.url) completed.") } catch { ILOG("GameImportQueue - processing item in queue: \(item.url) caught error...") @@ -360,11 +385,12 @@ public final class GameImporter: ObservableObject { private func performImport(for item: ImportQueueItem) async throws { - //detect type for updating UI - //todo: detect BIOS - if (isCDROM(item.url)) { + //detect type for updating UI and later processing + if (try isBIOS(item)) { //this can throw + item.fileType = .bios + } else if (isCDROM(item)) { item.fileType = .cdRom - } else if (isArtwork(item.url)) { + } else if (isArtwork(item)) { item.fileType = .artwork } else { item.fileType = .game @@ -372,23 +398,48 @@ public final class GameImporter: ObservableObject { var importedFiles: [URL] = [] - do { - if let importedFile = try await importSingleFile(at: item.url) { - importedFiles.append(importedFile) - } - } catch { - //TODO: what do i do here? - ELOG("Failed to import file at \(item.url): \(error.localizedDescription)") + //get valid systems that this object might support + guard let systems = try? await determineSystems(for: item), !systems.isEmpty else { + //this is actually an import error + item.status = .failure + throw GameImporterError.noSystemMatched } - await importedFiles.asyncForEach { path in - do { - try await self._handlePath(path: path, userChosenSystem: nil) - } catch { - //TODO: what do i do here? I could just let this throw or try and process what happened... - ELOG("\(error)") - } - } // for each + //this might be a conflict if we can't infer what to do + if item.systems.count > 1 { + //conflict + item.status = .conflict + //start figuring out what to do, because this item is a conflict + try await gameImporterFileService.moveToConflictsFolder(item, conflictsPath: conflictPath) + } + + //move ImportQueueItem to appropriate file location + try await gameImporterFileService.moveImportItem(toAppropriateSubfolder: item) + + //import the copied file into our database + + +// do { +// //try moving it to the correct location - we may clean this up later. +// if let importedFile = try await importSingleFile(at: item.url) { +// importedFiles.append(importedFile) +// } +// +// //try importing the moved file[s] into the Roms DB +// +// } catch { +// //TODO: what do i do here? +// ELOG("Failed to import file at \(item.url): \(error.localizedDescription)") +// } + +// await importedFiles.asyncForEach { path in +// do { +// try await self._handlePath(path: path, userChosenSystem: nil) +// } catch { +// //TODO: what do i do here? I could just let this throw or try and process what happened... +// ELOG("\(error)") +// } +// } // for each //external callers - might not be needed in the end self.completionHandler?(self.encounteredConflicts) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift new file mode 100644 index 0000000000..9ec519d4b1 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -0,0 +1,488 @@ +// +// File.swift +// PVLibrary +// +// Created by David Proskin on 11/5/24. +// + +import Foundation +import PVSupport +import RealmSwift +import PVCoreLoader +import AsyncAlgorithms +import PVPlists +import PVLookup +import Systems +import PVMediaCache +import PVFileSystem +import PVLogging +import PVPrimitives +import PVRealm +import Perception +import SwiftUI + +protocol GameImporterDatabaseServicing { + func setOpenVGDB(_ vgdb: OpenVGDB) + func setRomsPath(url:URL) + func importGameIntoDatabase(queueItem: ImportQueueItem) async throws +} + +class GameImporterDatabaseService : GameImporterDatabaseServicing { + static var charset: CharacterSet = { + var c = CharacterSet.punctuationCharacters + c.remove(charactersIn: ",-+&.'") + return c + }() + + var romsPath:URL? + var openVGDB: OpenVGDB? + init() { + + } + + func setRomsPath(url: URL) { + romsPath = url + } + + func setOpenVGDB(_ vgdb: OpenVGDB) { + openVGDB = vgdb + } + + internal func importGameIntoDatabase(queueItem: ImportQueueItem) async throws { + guard let targetSystem = queueItem.systems.first else { + throw GameImporterError.systemNotDetermined + } + + DLOG("Attempting to import game: \(queueItem.url.lastPathComponent) for system: \(targetSystem.name)") + + let filename = queueItem.url.lastPathComponent + let partialPath = (targetSystem.identifier as NSString).appendingPathComponent(filename) + let similarName = RomDatabase.altName(queueItem.url, systemIdentifier: targetSystem.identifier) + + DLOG("Checking game cache for partialPath: \(partialPath) or similarName: \(similarName)") + let gamesCache = RomDatabase.gamesCache + + if let existingGame = gamesCache[partialPath] ?? gamesCache[similarName], + targetSystem.identifier == existingGame.systemIdentifier { + DLOG("Found existing game in cache, saving relative path") + await saveRelativePath(existingGame, partialPath: partialPath, file: queueItem.url) + } else { + DLOG("No existing game found, starting import to database") + try await self.importToDatabaseROM(forItem: queueItem, system: targetSystem, relatedFiles: nil) + } + } + + /// Imports a ROM to the database + internal func importToDatabaseROM(forItem queueItem: ImportQueueItem, system: PVSystem, relatedFiles: [URL]?) async throws { + DLOG("Starting database ROM import for: \(queueItem.url.lastPathComponent)") + let filename = queueItem.url.lastPathComponent + let filenameSansExtension = queueItem.url.deletingPathExtension().lastPathComponent + let title: String = PVEmulatorConfiguration.stripDiscNames(fromFilename: filenameSansExtension) + let destinationDir = (system.identifier as NSString) + let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) + + DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") + let file = PVFile(withURL: queueItem.url) + let game = PVGame(withFile: file, system: system) + game.romPath = partialPath + game.title = title + game.requiresSync = true + var relatedPVFiles = [PVFile]() + let files = RomDatabase.getFileSystemROMCache(for: system).keys + let name = RomDatabase.altName(queueItem.url, systemIdentifier: system.identifier) + + DLOG("Searching for related files with name: \(name)") + + await files.asyncForEach { url in + let relativeName = RomDatabase.altName(url, systemIdentifier: system.identifier) + DLOG("Checking file \(url.lastPathComponent) with relative name: \(relativeName)") + if relativeName == name { + DLOG("Found matching related file: \(url.lastPathComponent)") + relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) + } + } + + if let relatedFiles = relatedFiles { + DLOG("Processing \(relatedFiles.count) additional related files") + for url in relatedFiles { + DLOG("Adding related file: \(url.lastPathComponent)") + relatedPVFiles.append(PVFile(withPartialPath: destinationDir.appendingPathComponent(url.lastPathComponent))) + } + } + + guard let md5 = calculateMD5(forGame: game)?.uppercased() else { + ELOG("Couldn't calculate MD5 for game \(partialPath)") + throw GameImporterError.couldNotCalculateMD5 + } + DLOG("Calculated MD5: \(md5)") + +// // Register import with coordinator +// guard await importCoordinator.checkAndRegisterImport(md5: md5) else { +// DLOG("Import already in progress for MD5: \(md5)") +// throw GameImporterError.romAlreadyExistsInDatabase +// } +// DLOG("Registered import with coordinator for MD5: \(md5)") + +// defer { +// Task { +// await importCoordinator.completeImport(md5: md5) +// DLOG("Completed import coordination for MD5: \(md5)") +// } +// } + + game.relatedFiles.append(objectsIn: relatedPVFiles) + game.md5Hash = md5 + try await finishUpdateOrImport(ofGame: game) + } + + /// Saves the relative path for a given game + func saveRelativePath(_ existingGame: PVGame, partialPath:String, file:URL) async { + if RomDatabase.gamesCache[partialPath] == nil { + await RomDatabase.addRelativeFileCache(file, game:existingGame) + } + } + + /// Finishes the update or import of a game + internal func finishUpdateOrImport(ofGame game: PVGame) async throws { + // Only process if rom doensn't exist in DB + if RomDatabase.gamesCache[game.romPath] != nil { + throw GameImporterError.romAlreadyExistsInDatabase + } + var modified = false + var game:PVGame = game + if game.requiresSync { +// if importStartedHandler != nil { +// let fullpath = PVEmulatorConfiguration.path(forGame: game) +// Task { @MainActor in +// self.importStartedHandler?(fullpath.path) +// } +// } + game = lookupInfo(for: game, overwrite: true) + modified = true + } + let wasModified = modified +// if finishedImportHandler != nil { +// let md5: String = game.md5Hash +// // Task { @MainActor in +// self.finishedImportHandler?(md5, wasModified) +// // } +// } + if game.originalArtworkFile == nil { + game = await getArtwork(forGame: game) + } + self.saveGame(game) + } + + @discardableResult + func getArtwork(forGame game: PVGame) async -> PVGame { + var url = game.originalArtworkURL + if url.isEmpty { + return game + } + if PVMediaCache.fileExists(forKey: url) { + if let localURL = PVMediaCache.filePath(forKey: url) { + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + game.originalArtworkFile = file + return game + } + } + DLOG("Starting Artwork download for \(url)") + // Note: Evil hack for bad domain in DB + url = url.replacingOccurrences(of: "gamefaqs1.cbsistatic.com/box/", with: "gamefaqs.gamespot.com/a/box/") + guard let artworkURL = URL(string: url) else { + ELOG("url is invalid url <\(url)>") + return game + } + let request = URLRequest(url: artworkURL) + var imageData:Data? + + if let response = try? await URLSession.shared.data(for: request), (response.1 as? HTTPURLResponse)?.statusCode == 200 { + imageData = response.0 + } + + if let data = imageData { +#if os(macOS) + if let artwork = NSImage(data: data) { + do { + let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + game.originalArtworkFile = file + } catch { ELOG("\(error.localizedDescription)") } + } +#elseif !os(watchOS) + if let artwork = UIImage(data: data) { + do { + let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + game.originalArtworkFile = file + } catch { ELOG("\(error.localizedDescription)") } + } +#endif + } + return game + } + + //MARK: Utility + + + @discardableResult + func lookupInfo(for game: PVGame, overwrite: Bool = true) -> PVGame { + game.requiresSync = false + if game.md5Hash.isEmpty { + if let romFullPath = romsPath?.appendingPathComponent(game.romPath).path { + if let md5Hash = calculateMD5(forGame: game) { + game.md5Hash = md5Hash + } + } + } + guard !game.md5Hash.isEmpty else { + NSLog("Game md5 has was empty") + return game + } + var resultsMaybe: [[String: Any]]? + do { + if let result = RomDatabase.getArtCache(game.md5Hash.uppercased(), systemIdentifier:game.systemIdentifier) { + resultsMaybe=[result] + } else { + resultsMaybe = try searchDatabase(usingKey: "romHashMD5", value: game.md5Hash.uppercased(), systemID: game.systemIdentifier) + } + } catch { + ELOG("\(error.localizedDescription)") + } + if resultsMaybe == nil || resultsMaybe!.isEmpty { //PVEmulatorConfiguration.supportedROMFileExtensions.contains(game.file.url.pathExtension.lowercased()) { + let fileName: String = game.file.url.lastPathComponent + // Remove any extraneous stuff in the rom name such as (U), (J), [T+Eng] etc + let nonCharRange: NSRange = (fileName as NSString).rangeOfCharacter(from: GameImporterDatabaseService.charset) + var gameTitleLen: Int + if nonCharRange.length > 0, nonCharRange.location > 1 { + gameTitleLen = nonCharRange.location - 1 + } else { + gameTitleLen = fileName.count + } + let subfileName = String(fileName.prefix(gameTitleLen)) + do { + if let result = RomDatabase.getArtCacheByFileName(subfileName, systemIdentifier:game.systemIdentifier) { + resultsMaybe=[result] + } else { + resultsMaybe = try searchDatabase(usingKey: "romFileName", value: subfileName, systemID: game.systemIdentifier) + } + } catch { + ELOG("\(error.localizedDescription)") + } + } + guard let results = resultsMaybe, !results.isEmpty else { + // the file maybe exists but was wiped from DB, + // try to re-import and rescan if can + // skip re-import during artwork download process + /* + let urls = importFiles(atPaths: [game.url]) + if !urls.isEmpty { + lookupInfo(for: game, overwrite: overwrite) + return + } else { + DLOG("Unable to find ROM \(game.romPath) in DB") + try? database.writeTransaction { + game.requiresSync = false + } + return + } + */ + return game + } + var chosenResultMaybe: [String: Any]? = + // Search by region id + results.first { (dict) -> Bool in + DLOG("region id: \(dict["regionID"] as? Int ?? 0)") + // Region ids USA = 21, Japan = 13 + return (dict["regionID"] as? Int) == 21 + } + ?? // If nothing, search by region string, could be a comma sepearted list + results.first { (dict) -> Bool in + DLOG("region: \(dict["region"] ?? "nil")") + // Region ids USA = 21, Japan = 13 + return (dict["region"] as? String)?.uppercased().contains("USA") ?? false + } + if chosenResultMaybe == nil { + if results.count > 1 { + ILOG("Query returned \(results.count) possible matches. Failed to matcha USA version by string or release ID int. Going to choose the first that exists in the DB.") + } + chosenResultMaybe = results.first + } + //write at the end of fininshOrUpdateImport + //autoreleasepool { + // do { + game.requiresSync = false + guard let chosenResult = chosenResultMaybe else { + NSLog("Unable to find ROM \(game.romPath) in OpenVGDB") + return game + } + /* Optional results + gameTitle + boxImageURL + region + gameDescription + boxBackURL + developer + publisher + year + genres [comma array string] + referenceURL + releaseID + regionID + systemShortName + serial + */ + if let title = chosenResult["gameTitle"] as? String, !title.isEmpty, overwrite || game.title.isEmpty { + // Remove just (Disc 1) from the title. Discs with other numbers will retain their names + let revisedTitle = title.replacingOccurrences(of: "\\ \\(Disc 1\\)", with: "", options: .regularExpression) + game.title = revisedTitle + } + + if let boxImageURL = chosenResult["boxImageURL"] as? String, !boxImageURL.isEmpty, overwrite || game.originalArtworkURL.isEmpty { + game.originalArtworkURL = boxImageURL + } + + if let regionName = chosenResult["region"] as? String, !regionName.isEmpty, overwrite || game.regionName == nil { + game.regionName = regionName + } + + if let regionID = chosenResult["regionID"] as? Int, overwrite || game.regionID.value == nil { + game.regionID.value = regionID + } + + if let gameDescription = chosenResult["gameDescription"] as? String, !gameDescription.isEmpty, overwrite || game.gameDescription == nil { + let options = [NSAttributedString.DocumentReadingOptionKey.documentType: NSAttributedString.DocumentType.html] + if let data = gameDescription.data(using: .isoLatin1) { + do { + let htmlDecodedGameDescription = try NSMutableAttributedString(data: data, options: options, documentAttributes: nil) + game.gameDescription = htmlDecodedGameDescription.string.replacingOccurrences(of: "(\\.|\\!|\\?)([A-Z][A-Za-z\\s]{2,})", with: "$1\n\n$2", options: .regularExpression) + } catch { + ELOG("\(error.localizedDescription)") + } + } + } + + if let boxBackURL = chosenResult["boxBackURL"] as? String, !boxBackURL.isEmpty, overwrite || game.boxBackArtworkURL == nil { + game.boxBackArtworkURL = boxBackURL + } + + if let developer = chosenResult["developer"] as? String, !developer.isEmpty, overwrite || game.developer == nil { + game.developer = developer + } + + if let publisher = chosenResult["publisher"] as? String, !publisher.isEmpty, overwrite || game.publisher == nil { + game.publisher = publisher + } + + if let genres = chosenResult["genres"] as? String, !genres.isEmpty, overwrite || game.genres == nil { + game.genres = genres + } + + if let releaseDate = chosenResult["releaseDate"] as? String, !releaseDate.isEmpty, overwrite || game.publishDate == nil { + game.publishDate = releaseDate + } + + if let referenceURL = chosenResult["referenceURL"] as? String, !referenceURL.isEmpty, overwrite || game.referenceURL == nil { + game.referenceURL = referenceURL + } + + if let releaseID = chosenResult["releaseID"] as? NSNumber, !releaseID.stringValue.isEmpty, overwrite || game.releaseID == nil { + game.releaseID = releaseID.stringValue + } + + if let systemShortName = chosenResult["systemShortName"] as? String, !systemShortName.isEmpty, overwrite || game.systemShortName == nil { + game.systemShortName = systemShortName + } + + if let romSerial = chosenResult["serial"] as? String, !romSerial.isEmpty, overwrite || game.romSerial == nil { + game.romSerial = romSerial + } + // } catch { + // ELOG("Failed to update game \(game.title) : \(error.localizedDescription)") + // } + //} + return game + } + + func releaseID(forCRCs crcs: Set) -> String? { + return openVGDB?.releaseID(forCRCs: crcs) + } + + enum DatabaseQueryError: Error { + case invalidSystemID + } + + func searchDatabase(usingKey key: String, value: String, systemID: SystemIdentifier) throws -> [[String: NSObject]]? { + guard let systemIDInt = PVEmulatorConfiguration.databaseID(forSystemID: systemID.rawValue) else { + throw DatabaseQueryError.invalidSystemID + } + + return try openVGDB?.searchDatabase(usingKey: key, value: value, systemID: systemIDInt) + } + + func searchDatabase(usingKey key: String, value: String, systemID: String) throws -> [[String: NSObject]]? { + guard let systemIDInt = PVEmulatorConfiguration.databaseID(forSystemID: systemID) else { + throw DatabaseQueryError.invalidSystemID + } + + return try openVGDB?.searchDatabase(usingKey: key, value: value, systemID: systemIDInt) + } + + // TODO: This was a quick copy of the general version for filenames specifically + func searchDatabase(usingFilename filename: String, systemID: String) throws -> [[String: NSObject]]? { + guard let systemIDInt = PVEmulatorConfiguration.databaseID(forSystemID: systemID) else { + throw DatabaseQueryError.invalidSystemID + } + + return try openVGDB?.searchDatabase(usingFilename: filename, systemID: systemIDInt) + } + func searchDatabase(usingFilename filename: String, systemIDs: [String]) throws -> [[String: NSObject]]? { + let systemIDsInts: [Int] = systemIDs.compactMap { PVEmulatorConfiguration.databaseID(forSystemID: $0) } + guard !systemIDsInts.isEmpty else { + throw DatabaseQueryError.invalidSystemID + } + + return try openVGDB?.searchDatabase(usingFilename: filename, systemIDs: systemIDsInts) + } + + /// Saves a game to the database + func saveGame(_ game:PVGame) { + do { + let database = RomDatabase.sharedInstance + try database.writeTransaction { + database.realm.create(PVGame.self, value:game, update:.modified) + } + RomDatabase.addGamesCache(game) + } catch { + ELOG("Couldn't add new game \(error.localizedDescription)") + } + } + + /// Calculates the MD5 hash for a given game + @objc + public func calculateMD5(forGame game: PVGame) -> String? { + var offset: UInt64 = 0 + + //this seems to be spread in many places, not sure why. it might be doable to put this in the queue item, but for now, trying to consolidate. + //I have no history or explanation for why we need the 16 offset for SNES/NES + if game.systemIdentifier == SystemIdentifier.SNES.rawValue { + offset = SystemIdentifier.SNES.offset + } else if game.systemIdentifier == SystemIdentifier.NES.rawValue { + offset = SystemIdentifier.NES.offset + } else if let system = SystemIdentifier(rawValue: game.systemIdentifier) { + offset = system.offset + } + + let romPath = romsPath?.appendingPathComponent(game.romPath, isDirectory: false) + if let romPath = romPath { + let fm = FileManager.default + if !fm.fileExists(atPath: romPath.path) { + ELOG("Cannot find file at path: \(romPath)") + return nil + } + return fm.md5ForFile(atPath: romPath.path, fromOffset: offset) + } + + return nil + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift new file mode 100644 index 0000000000..261c2c497c --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -0,0 +1,207 @@ +// +// File.swift +// PVLibrary +// +// Created by David Proskin on 11/5/24. +// + +import Foundation +import PVSupport +import RealmSwift + +protocol GameImporterFileServicing { + func moveImportItem(toAppropriateSubfolder queueItem: ImportQueueItem) async throws + func moveToConflictsFolder(_ queueItem: ImportQueueItem, conflictsPath: URL) async throws +} + +class GameImporterFileService : GameImporterFileServicing { + + init() { + + } + + package func moveImportItem(toAppropriateSubfolder queueItem: ImportQueueItem) async throws { + switch (queueItem.fileType) { + + case .bios: + _ = try await handleBIOSItem(queueItem) + case .artwork: + //TODO: implement me + return + case .game: + _ = try await handleROM(queueItem) + return + case .cdRom: + _ = try await handleCDROMItem(queueItem) + case .unknown: + throw GameImporterError.unsupportedFile + } + } + + // MARK: - BIOS + + /// Ensures a BIOS file is copied to appropriate file destinations + /// Returns the array of PVSystems that this BIOS file applies to + private func handleBIOSItem(_ queueItem: ImportQueueItem) async throws { + guard queueItem.fileType == .bios, let md5 = queueItem.md5?.uppercased() else { + throw GameImporterError.couldNotCalculateMD5 + } + + let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) + + guard !biosMatches.isEmpty else { + //we shouldn't be here. + throw GameImporterError.noBIOSMatchForBIOSFileType + } + + for bios in biosMatches { + if let system = bios.system { + DLOG("Copying BIOS to system: \(system.name)") + let biosPath = PVEmulatorConfiguration.biosPath(forSystemIdentifier: system.identifier) + .appendingPathComponent(bios.expectedFilename) + + queueItem.destinationUrl = try await moveFile(queueItem.url, to: biosPath) + } + } + } + //MARK: - Normal ROMs + + /// Moves a ROM to the appropriate subfolder + internal func handleROM(_ queueItem: ImportQueueItem) async throws { + guard queueItem.fileType == .game else { + throw GameImporterError.unsupportedFile + } + + guard !queueItem.systems.isEmpty else { + throw GameImporterError.noSystemMatched + } + + let candidateSystems = queueItem.systems + if (candidateSystems.count > 1 && queueItem.userChosenSystem == nil) { + //conflict? + throw GameImporterError.systemNotDetermined + } + + let fileManager = FileManager.default + let fileName = queueItem.url.lastPathComponent + + if let system = candidateSystems.first { + let destinationFolder = system.romsDirectory + let destinationPath = destinationFolder.appendingPathComponent(fileName) + queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) + } + } + + // MARK: - CDROM + + /// Ensures a BIOS file is copied to appropriate file destinations + /// Returns the array of PVSystems that this BIOS file applies to + private func handleCDROMItem(_ queueItem: ImportQueueItem) async throws { + guard queueItem.fileType == .cdRom else { + throw GameImporterError.unsupportedCDROMFile + } + + guard !queueItem.systems.isEmpty else { + throw GameImporterError.unsupportedCDROMFile + } + + let candidateSystems = queueItem.systems + if (candidateSystems.count > 1 && queueItem.userChosenSystem == nil) { + //conflict? + throw GameImporterError.systemNotDetermined + } + + let fileManager = FileManager.default + let fileName = queueItem.url.lastPathComponent + + if let system = candidateSystems.first { + let destinationFolder = system.romsDirectory + let destinationPath = destinationFolder.appendingPathComponent(fileName) + + // Check for M3U + //TODO: implement handling for M3U and cue files + #if false + if let system = try await handleM3UFile(candidate) { + DLOG("Moving M3U and referenced files to system: \(system.name)") + // Move M3U and all referenced files to system directory + let destinationDir = system.romsDirectory + return try await moveM3UAndReferencedFiles(candidate, to: destinationDir) + } + + // If cue file, try to match its bin file + if `extension` == "cue" { + if let binFile = try findAssociatedBinFile(for: candidate) { + DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") + let binCandidate = ImportCandidateFile(filePath: binFile) + if let system = try? await determineSystem(for: binCandidate) { + DLOG("Found system match from associated bin file: \(system.name)") + return system + } + } + } + #endif + + do { + queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) + } catch { + throw GameImporterError.failedToMoveCDROM(error) + } + } + } + + private func findAssociatedBinFile(for cueFileItem: ImportQueueItem) throws -> URL? { + let cueContents = try String(contentsOf: cueFileItem.url, encoding: .utf8) + let lines = cueContents.components(separatedBy: .newlines) + + // Look for FILE "something.bin" BINARY line + for line in lines { + let components = line.trimmingCharacters(in: .whitespaces) + .components(separatedBy: "\"") + guard components.count >= 2, + line.lowercased().contains("file") && line.lowercased().contains("binary") else { + continue + } + + let binFileName = components[1] + let binPath = cueFileItem.url.deletingLastPathComponent().appendingPathComponent(binFileName) + + if FileManager.default.fileExists(atPath: binPath.path) { + return binPath + } + } + + return nil + } + + // MARK: - Utility + + + /// Moves a file to the conflicts directory + internal func moveToConflictsFolder(_ queueItem: ImportQueueItem, conflictsPath: URL) async throws { + let destination = conflictsPath.appendingPathComponent(queueItem.url.lastPathComponent) + queueItem.destinationUrl = try moveAndOverWrite(sourcePath: queueItem.url, destinationPath: destination) + } + + /// Move a `URL` to a destination, creating the destination directory if needed + private func moveFile(_ file: URL, to destination: URL) async throws -> URL { + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let destPath = destination.appendingPathComponent(file.lastPathComponent) + try FileManager.default.moveItem(at: file, to: destPath) + DLOG("Moved file to: \(destPath.path)") + return destPath + } + + /// Moves a file and overwrites if it already exists at the destination + public func moveAndOverWrite(sourcePath: URL, destinationPath: URL) throws -> URL { + let fileManager = FileManager.default + + // If file exists at destination, remove it first + if fileManager.fileExists(atPath: destinationPath.path) { + try fileManager.removeItem(at: destinationPath) + } + + // Now move the file + try fileManager.moveItem(at: sourcePath, to: destinationPath) + return destinationPath + } +} diff --git a/PVLibrary/Sources/Systems/SystemIdentifier.swift b/PVLibrary/Sources/Systems/SystemIdentifier.swift index 3a5b501615..9cb20bc1b7 100644 --- a/PVLibrary/Sources/Systems/SystemIdentifier.swift +++ b/PVLibrary/Sources/Systems/SystemIdentifier.swift @@ -210,6 +210,7 @@ public enum SystemIdentifier: String, CaseIterable, Codable { var offset: UInt64 { switch self { case .SNES: return 16 + case .NES: return 16 default: return 0 } } diff --git a/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift b/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift new file mode 100644 index 0000000000..ad448369bc --- /dev/null +++ b/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift @@ -0,0 +1,32 @@ +// +// GameImporterTests.swift +// PVLibrary +// +// Created by David Proskin on 11/4/24. +// + +@testable import PVLibrary +import XCTest + +class GameImporterTests: XCTestCase { + + var gameImporter: GameImporting! + + override func setUp() async throws { + try await super.setUp() + // Put setup code here. This method is called before the invocation of each test method in the class. + gameImporter = GameImporter(FileManager.default) +// await gameImporter.initSystems() <--- this will crash until we get proper DI + } + + override func tearDown() { + // Put teardown code here. This method is called after the invocation of each test method in the class. + super.tearDown() + gameImporter = nil + } + + func testImportSingleGame_Success() { + // Arrange + let testData = "Test Game Data".data(using: .utf8) + } +} diff --git a/PVLibrary/Tests/PVLibraryTests/PVLibraryTests.swift b/PVLibrary/Tests/PVLibraryTests/PVLibraryTests.swift index 3dad8e3c77..55d99a2227 100644 --- a/PVLibrary/Tests/PVLibraryTests/PVLibraryTests.swift +++ b/PVLibrary/Tests/PVLibraryTests/PVLibraryTests.swift @@ -14,7 +14,7 @@ class PVLibraryTests: XCTestCase { super.setUp() // Put setup code here. This method is called before the invocation of each test method in the class. try! FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true) - try! RomDatabase.initDefaultDatabase() +// try! RomDatabase.initDefaultDatabase() } override func tearDown() { From a2c602a9b0c1aa94bdab73799f009e18b1dc8356 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Wed, 6 Nov 2024 16:26:25 -0500 Subject: [PATCH 04/30] Adds a construct for ImportQueueItem that will support CDROMs - an ImportQueueItem can now contain Children. We also now check the queue (and any child items in the queue) for duplicates. next step is to pre-process the queue to fix up or detect CDRoms properly. --- .../Importer/Models/ImportQueueItem.swift | 4 ++++ .../Services/GameImporter/GameImporter.swift | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift index 12d1cce64e..e0032ef676 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift @@ -58,6 +58,9 @@ public class ImportQueueItem: Identifiable, ObservableObject { public var userChosenSystem: System? public var destinationUrl: URL? + //this is used when a single import has child items - e.g., m3u, cue, directory + public var childQueueItems:[ImportQueueItem] + // Observable status for individual imports public var status: ImportStatus = .queued @@ -66,6 +69,7 @@ public class ImportQueueItem: Identifiable, ObservableObject { self.fileType = fileType self.systems = [] self.userChosenSystem = nil + self.childQueueItems = [] } public var md5: String? { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 5655dbcf18..f574e9c0f8 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -452,12 +452,24 @@ public final class GameImporter: GameImporting, ObservableObject { } } - private func addImportItemToQueue(_ item: ImportQueueItem) { + /// Checks the queue and all child elements in the queue to see if this file exists. if it does, return true, else return false. + private func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool { let duplicate = importQueue.contains { existing in - return (existing.url == item.url || existing.id == item.id) ? true : false + var exists = false + if (existing.url == queueItem.url || existing.id == queueItem.id) { + return true + } else if (!existing.childQueueItems.isEmpty) { + //check the child queue items for duplicates + return self.importQueueContainsDuplicate(existing.childQueueItems, ofItem: queueItem) + } + return false } - guard !duplicate else { + return duplicate + } + + private func addImportItemToQueue(_ item: ImportQueueItem) { + guard !importQueueContainsDuplicate(self.importQueue, ofItem: item) else { WLOG("GameImportQueue - Trying to add duplicate ImportItem to import queue with url: \(item.url) and id: \(item.id)") return; } From 8294179818913f463b733c16248d1c07fcf334c9 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Wed, 6 Nov 2024 21:56:26 -0500 Subject: [PATCH 05/30] implement queue pre-sort functions that ensure proper parenting to related files, if they're in the queue. this will make it easy to move files to the right place all at once later in the process --- .../PVEmulatorConfiguration.swift | 62 ------ .../Services/GameImporter/GameImporter.swift | 191 +++++++++++++++++- .../GameImporterFileService.swift | 80 ++------ 3 files changed, 194 insertions(+), 139 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Configuration/PVEmulatorConfiguration.swift b/PVLibrary/Sources/PVLibrary/Configuration/PVEmulatorConfiguration.swift index bc0a9624c5..a5db11ada6 100644 --- a/PVLibrary/Sources/PVLibrary/Configuration/PVEmulatorConfiguration.swift +++ b/PVLibrary/Sources/PVLibrary/Configuration/PVEmulatorConfiguration.swift @@ -365,68 +365,6 @@ public extension PVEmulatorConfiguration { } } } - - class func cmpSpecialExt(obj1Extension: String, obj2Extension: String) -> Bool { - if obj1Extension == "m3u" && obj2Extension != "m3u" { - return obj1Extension > obj2Extension - } else if obj1Extension == "m3u" { - return false - } else if obj2Extension == "m3u" { - return true - } - if Extensions.artworkExtensions.contains(obj1Extension) { - return false - } else if Extensions.artworkExtensions.contains(obj2Extension) { - return true - } - return obj1Extension > obj2Extension - } - - class func cmp(obj1: URL, obj2: URL) -> Bool { - let obj1Filename = obj1.lastPathComponent - let obj2Filename = obj2.lastPathComponent - let obj1Extension = obj1.pathExtension.lowercased() - let obj2Extension = obj2.pathExtension.lowercased() - let name1=PVEmulatorConfiguration.stripDiscNames(fromFilename: obj1Filename) - let name2=PVEmulatorConfiguration.stripDiscNames(fromFilename: obj2Filename) - if name1 == name2 { - // Standard sort - if obj1Extension == obj2Extension { - return obj1Filename < obj2Filename - } - return obj1Extension > obj2Extension - } else { - return name1 < name2 - } - } - - class func sortImportURLs(urls: [URL]) -> [URL] { - var ext:[String:[URL]] = [:] - // separate array by file extension - urls.forEach({ (url) in - if var urls = ext[url.pathExtension.lowercased()] { - urls.append(url) - ext[url.pathExtension.lowercased()]=urls - } else { - ext[url.pathExtension.lowercased()]=[url] - } - }) - // sort - var sorted: [URL] = [] - ext.keys - .sorted(by: cmpSpecialExt) - .forEach { - if let values = ext[$0] { - let values = values.sorted { (obj1, obj2) -> Bool in - return cmp(obj1: obj1, obj2: obj2) - } - sorted.append(contentsOf: values) - ext[$0] = values - } - } - VLOG(sorted.map { $0.lastPathComponent }.joined(separator: ", ")) - return sorted - } } // MARK: System queries diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index f574e9c0f8..4c09ac7678 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -314,7 +314,7 @@ public final class GameImporter: GameImporting, ObservableObject { notificationToken?.invalidate() } - //MARK: Queue Management + //MARK: Public Queue Management // Adds an ImportItem to the queue without starting processing public func addImport(_ item: ImportQueueItem) { @@ -339,6 +339,176 @@ public final class GameImporter: GameImporting, ObservableObject { await processQueue() } } + + //MARK: Processing functions + + private func preProcessQueue() async { + //determine the type for all items in the queue + for importItem in self.importQueue { + //ideally this wouldn't be needed here + do { + importItem.fileType = try determineImportType(importItem) + } catch { + //caught an error trying to assign file type + } + + } + + //sort the queue to make sure m3us go first + importQueue = sortImportQueueItems(importQueue) + + //thirdly, we need to parse the queue and find any children for cue files + organizeCueAndBinFiles(in: &importQueue) + + //lastly, move and cue (and child bin) files under the parent m3u (if they exist) + organizeM3UFiles(in: &importQueue) + } + + internal func organizeM3UFiles(in importQueue: inout [ImportQueueItem]) { + + for m3uitem in importQueue where m3uitem.url.pathExtension.lowercased() == "m3u" { + let baseFileName = m3uitem.url.deletingPathExtension().lastPathComponent + + do { + let contents = try String(contentsOf: m3uitem.url, encoding: .utf8) + let files = contents.components(separatedBy: .newlines) + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && !$0.hasPrefix("#") } + + // Move all referenced files + for filename in files { + if let cueIndex = importQueue.firstIndex(where: { item in + item.url.lastPathComponent == filename + }) { + // Remove the .bin item from the queue and add it as a child of the .cue item + let cueItem = importQueue.remove(at: cueIndex) + cueItem.fileType = .cdRom + m3uitem.childQueueItems.append(cueItem) + } + } + } catch { + ELOG("Caught an error looking for a corresponding .cues to \(baseFileName) - probably bad things happening") + } + } + } + + // Function to process ImportQueueItems and associate .bin files with corresponding .cue files + internal func organizeCueAndBinFiles(in importQueue: inout [ImportQueueItem]) { + // Loop through a copy of the queue to avoid mutation issues while iterating + for cueItem in importQueue where cueItem.url.pathExtension.lowercased() == "cue" { + // Extract the base name of the .cue file (without extension) + let baseFileName = cueItem.url.deletingPathExtension().lastPathComponent + + do { + if let candidateBinUrl = try self.findAssociatedBinFile(for: cueItem) { + // Find any .bin item in the queue that matches the .cue base file name + if let binIndex = importQueue.firstIndex(where: { item in + item.url == candidateBinUrl + }) { + // Remove the .bin item from the queue and add it as a child of the .cue item + let binItem = importQueue.remove(at: binIndex) + binItem.fileType = .cdRom + cueItem.childQueueItems.append(binItem) + } + } else { + //this is probably some kind of error... + ELOG("Found a .cue \(baseFileName) without a .bin - probably bad things happening") + } + } catch { + ELOG("Caught an error looking for a corresponding .bin to \(baseFileName) - probably bad things happening") + } + } + } + + private func findAssociatedBinFile(for cueFileItem: ImportQueueItem) throws -> URL? { + let cueContents = try String(contentsOf: cueFileItem.url, encoding: .utf8) + let lines = cueContents.components(separatedBy: .newlines) + + // Look for FILE "something.bin" BINARY line + for line in lines { + let components = line.trimmingCharacters(in: .whitespaces) + .components(separatedBy: "\"") + guard components.count >= 2, + line.lowercased().contains("file") && line.lowercased().contains("binary") else { + continue + } + + let binFileName = components[1] + let binPath = cueFileItem.url.deletingLastPathComponent().appendingPathComponent(binFileName) + + if FileManager.default.fileExists(atPath: binPath.path) { + return binPath + } + } + + return nil + } + + + internal func cmpSpecialExt(obj1Extension: String, obj2Extension: String) -> Bool { + if obj1Extension == "m3u" && obj2Extension != "m3u" { + return obj1Extension > obj2Extension + } else if obj1Extension == "m3u" { + return false + } else if obj2Extension == "m3u" { + return true + } + if Extensions.artworkExtensions.contains(obj1Extension) { + return false + } else if Extensions.artworkExtensions.contains(obj2Extension) { + return true + } + return obj1Extension > obj2Extension + } + + internal func cmp(obj1: ImportQueueItem, obj2: ImportQueueItem) -> Bool { + let url1 = obj1.url + let url2 = obj2.url + let obj1Filename = url1.lastPathComponent + let obj2Filename = url2.lastPathComponent + let obj1Extension = url1.pathExtension.lowercased() + let obj2Extension = url2.pathExtension.lowercased() + let name1=PVEmulatorConfiguration.stripDiscNames(fromFilename: obj1Filename) + let name2=PVEmulatorConfiguration.stripDiscNames(fromFilename: obj2Filename) + if name1 == name2 { + // Standard sort + if obj1Extension == obj2Extension { + return obj1Filename < obj2Filename + } + return obj1Extension > obj2Extension + } else { + return name1 < name2 + } + } + + internal func sortImportQueueItems(_ importQueueItems: [ImportQueueItem]) -> [ImportQueueItem] { + var ext:[String:[ImportQueueItem]] = [:] + // separate array by file extension + importQueueItems.forEach({ (queueItem) in + let fileExt = queueItem.url.pathExtension.lowercased() + if var itemsWithExtension = ext[fileExt] { + itemsWithExtension.append(queueItem) + ext[fileExt]=itemsWithExtension + } else { + ext[fileExt]=[queueItem] + } + }) + // sort + var sorted: [ImportQueueItem] = [] + ext.keys + .sorted(by: cmpSpecialExt) + .forEach { + if let values = ext[$0] { + let values = values.sorted { (obj1, obj2) -> Bool in + return cmp(obj1: obj1, obj2: obj2) + } + sorted.append(contentsOf: values) + ext[$0] = values + } + } + VLOG(sorted.map { $0.url.lastPathComponent }.joined(separator: ", ")) + return sorted + } // Processes each ImportItem in the queue sequentially private func processQueue() async { @@ -383,20 +553,23 @@ public final class GameImporter: GameImporting, ObservableObject { } } - private func performImport(for item: ImportQueueItem) async throws { - + private func determineImportType(_ item: ImportQueueItem) throws -> FileType { //detect type for updating UI and later processing if (try isBIOS(item)) { //this can throw - item.fileType = .bios + return .bios } else if (isCDROM(item)) { - item.fileType = .cdRom + return .cdRom } else if (isArtwork(item)) { - item.fileType = .artwork + return .artwork } else { - item.fileType = .game + return .game } + } + + private func performImport(for item: ImportQueueItem) async throws { - var importedFiles: [URL] = [] + //ideally this wouldn't be needed here + item.fileType = try determineImportType(item) //get valid systems that this object might support guard let systems = try? await determineSystems(for: item), !systems.isEmpty else { @@ -452,7 +625,7 @@ public final class GameImporter: GameImporting, ObservableObject { } } - /// Checks the queue and all child elements in the queue to see if this file exists. if it does, return true, else return false. + /// Checks the queue and all child elements in the queue to see if this file exists. if it does, return true, else return false. private func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool { let duplicate = importQueue.contains { existing in var exists = false diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index 261c2c497c..dc0f6d743c 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -76,20 +76,15 @@ class GameImporterFileService : GameImporterFileServicing { throw GameImporterError.noSystemMatched } - let candidateSystems = queueItem.systems - if (candidateSystems.count > 1 && queueItem.userChosenSystem == nil) { - //conflict? + guard let targetSystem = queueItem.targetSystem() else { throw GameImporterError.systemNotDetermined } - let fileManager = FileManager.default let fileName = queueItem.url.lastPathComponent + let destinationFolder = targetSystem.romsDirectory + let destinationPath = destinationFolder.appendingPathComponent(fileName) - if let system = candidateSystems.first { - let destinationFolder = system.romsDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) - } + queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) } // MARK: - CDROM @@ -105,72 +100,21 @@ class GameImporterFileService : GameImporterFileServicing { throw GameImporterError.unsupportedCDROMFile } - let candidateSystems = queueItem.systems - if (candidateSystems.count > 1 && queueItem.userChosenSystem == nil) { - //conflict? + guard let targetSystem = queueItem.targetSystem() else { throw GameImporterError.systemNotDetermined } - let fileManager = FileManager.default let fileName = queueItem.url.lastPathComponent + let destinationFolder = targetSystem.romsDirectory + let destinationPath = destinationFolder.appendingPathComponent(fileName) - if let system = candidateSystems.first { - let destinationFolder = system.romsDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - - // Check for M3U - //TODO: implement handling for M3U and cue files - #if false - if let system = try await handleM3UFile(candidate) { - DLOG("Moving M3U and referenced files to system: \(system.name)") - // Move M3U and all referenced files to system directory - let destinationDir = system.romsDirectory - return try await moveM3UAndReferencedFiles(candidate, to: destinationDir) - } - - // If cue file, try to match its bin file - if `extension` == "cue" { - if let binFile = try findAssociatedBinFile(for: candidate) { - DLOG("Found associated bin file, trying to match: \(binFile.lastPathComponent)") - let binCandidate = ImportCandidateFile(filePath: binFile) - if let system = try? await determineSystem(for: binCandidate) { - DLOG("Found system match from associated bin file: \(system.name)") - return system - } - } - } - #endif - - do { - queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) - } catch { - throw GameImporterError.failedToMoveCDROM(error) - } - } - } - - private func findAssociatedBinFile(for cueFileItem: ImportQueueItem) throws -> URL? { - let cueContents = try String(contentsOf: cueFileItem.url, encoding: .utf8) - let lines = cueContents.components(separatedBy: .newlines) + //TODO: check and process children - // Look for FILE "something.bin" BINARY line - for line in lines { - let components = line.trimmingCharacters(in: .whitespaces) - .components(separatedBy: "\"") - guard components.count >= 2, - line.lowercased().contains("file") && line.lowercased().contains("binary") else { - continue - } - - let binFileName = components[1] - let binPath = cueFileItem.url.deletingLastPathComponent().appendingPathComponent(binFileName) - - if FileManager.default.fileExists(atPath: binPath.path) { - return binPath - } + do { + queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) + } catch { + throw GameImporterError.failedToMoveCDROM(error) } - - return nil } // MARK: - Utility From 73dce6031b274221cd6c613210c78709fcf4f5e3 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 12:32:43 -0500 Subject: [PATCH 06/30] bunch more cleanup, extraction to some smaller classes. almost there for being able to test a basic import through the system --- .../GameImporter/GameImporter+Artwork.swift | 366 +++++++++--------- .../GameImporter/GameImporter+ROMLookup.swift | 16 - .../Services/GameImporter/GameImporter.swift | 20 +- .../GameImporterDatabaseService.swift | 239 ++++++------ .../GameImporterFileService.swift | 64 ++- ...swift => GameImporterSystemsService.swift} | 44 ++- 6 files changed, 370 insertions(+), 379 deletions(-) delete mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift rename PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/{GameImporter+Systems.swift => GameImporterSystemsService.swift} (93%) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift index 8eeaa0e9a4..029240962b 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift @@ -8,186 +8,186 @@ public extension GameImporter { /// Imports artwork from a given path - class func importArtwork(fromPath imageFullPath: URL) async -> PVGame? { - var isDirectory: ObjCBool = false - let fileExists = FileManager.default.fileExists(atPath: imageFullPath.path, isDirectory: &isDirectory) - if !fileExists || isDirectory.boolValue { - WLOG("File doesn't exist or is directory at \(imageFullPath)") - return nil - } - - var success = false - - defer { - if success { - do { - try FileManager.default.removeItem(at: imageFullPath) - } catch { - ELOG("Failed to delete image at path \(imageFullPath) \n \(error.localizedDescription)") - } - } - } - - let gameFilename: String = imageFullPath.deletingPathExtension().lastPathComponent - let gameExtension = imageFullPath.deletingPathExtension().pathExtension - let database = RomDatabase.sharedInstance - - if gameExtension.isEmpty { - ILOG("Trying to import artwork that didn't contain the extension of the system") - let games = database.all(PVGame.self, filter: NSPredicate(format: "romPath CONTAINS[c] %@", argumentArray: [gameFilename])) - - if games.count == 1, let game = games.first { - ILOG("File for image didn't have extension for system but we found a single match for image \(imageFullPath.lastPathComponent) to game \(game.title) on system \(game.systemIdentifier)") - guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { - return nil - } - - do { - try database.writeTransaction { - game.customArtworkURL = hash - } - success = true - ILOG("Set custom artwork of game \(game.title) from file \(imageFullPath.lastPathComponent)") - } catch { - ELOG("Couldn't update game \(game.title) with new artwork URL \(hash)") - } - - return game - } else { - VLOG("Database search returned \(games.count) results") - } - } - - guard let systems: [PVSystem] = PVEmulatorConfiguration.systemsFromCache(forFileExtension: gameExtension), !systems.isEmpty else { - ELOG("No system for extension \(gameExtension)") - return nil - } - - let cdBasedSystems = PVEmulatorConfiguration.cdBasedSystems - let couldBelongToCDSystem = !Set(cdBasedSystems).isDisjoint(with: Set(systems)) - - if (couldBelongToCDSystem && PVEmulatorConfiguration.supportedCDFileExtensions.contains(gameExtension.lowercased())) || systems.count > 1 { - guard let existingGames = findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: gameFilename) else { - ELOG("System for extension \(gameExtension) is a CD system and {\(gameExtension)} not the right matching file type of cue or m3u") - return nil - } - if existingGames.count == 1, let onlyMatch = existingGames.first { - ILOG("We found a hit for artwork that could have been belonging to multiple games and only found one file that matched by systemid/filename. The winner is \(onlyMatch.title) for \(onlyMatch.systemIdentifier)") - - guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { - ELOG("Couldn't move image, fail to set custom artwork") - return nil - } - - do { - try database.writeTransaction { - onlyMatch.customArtworkURL = hash - } - success = true - } catch { - ELOG("Couldn't update game \(onlyMatch.title) with new artwork URL") - } - return onlyMatch - } else { - ELOG("We got to the unlikely scenario where an extension is possibly a CD binary file, or belongs to a system, and had multiple games that matched the filename under more than one core.") - return nil - } - } - - guard let system = systems.first else { - ELOG("systems empty") - return nil - } - - var gamePartialPath: String = URL(fileURLWithPath: system.identifier, isDirectory: true).appendingPathComponent(gameFilename).deletingPathExtension().path - if gamePartialPath.first == "/" { - gamePartialPath.removeFirst() - } - - if gamePartialPath.isEmpty { - ELOG("Game path was empty") - return nil - } - - var games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), value: gamePartialPath) - if games.isEmpty { - games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), beginsWith: gamePartialPath) - } - - guard !games.isEmpty else { - ELOG("Couldn't find game for path \(gamePartialPath)") - return nil - } - - if games.count > 1 { - WLOG("There were multiple matches for \(gamePartialPath)! #\(games.count). Going with first for now until we make better code to prompt user.") - } - - let game = games.first! - - guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { - ELOG("scaleAndMoveImageToCache failed") - return nil - } - - do { - try database.writeTransaction { - game.customArtworkURL = hash - } - success = true - } catch { - ELOG("Couldn't update game with new artwork URL") - } - - return game - } - - /// Scales and moves an image to the cache - fileprivate class func scaleAndMoveImageToCache(imageFullPath: URL) -> String? { - let coverArtFullData: Data - do { - coverArtFullData = try Data(contentsOf: imageFullPath, options: []) - } catch { - ELOG("Couldn't read data from image file \(imageFullPath.path)\n\(error.localizedDescription)") - return nil - } - -#if canImport(UIKit) - guard let coverArtFullImage = UIImage(data: coverArtFullData) else { - ELOG("Failed to create Image from data") - return nil - } - guard let coverArtScaledImage = coverArtFullImage.scaledImage(withMaxResolution: Int(PVThumbnailMaxResolution)) else { - ELOG("Failed to create scale image") - return nil - } -#else - guard let coverArtFullImage = NSImage(data: coverArtFullData) else { - ELOG("Failed to create Image from data") - return nil - } - let coverArtScaledImage = coverArtFullImage -#endif - -#if canImport(UIKit) - guard let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) else { - ELOG("Failed to create data representation of scaled image") - return nil - } -#else - let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) -#endif - - let hash: String = (coverArtScaledData as NSData).md5 - - do { - let destinationURL = try PVMediaCache.writeData(toDisk: coverArtScaledData, withKey: hash) - VLOG("Scaled and moved image from \(imageFullPath.path) to \(destinationURL.path)") - } catch { - ELOG("Failed to save artwork to cache: \(error.localizedDescription)") - return nil - } - - return hash - } -} \ No newline at end of file +// class func importArtwork(fromPath imageFullPath: URL) async -> PVGame? { +// var isDirectory: ObjCBool = false +// let fileExists = FileManager.default.fileExists(atPath: imageFullPath.path, isDirectory: &isDirectory) +// if !fileExists || isDirectory.boolValue { +// WLOG("File doesn't exist or is directory at \(imageFullPath)") +// return nil +// } +// +// var success = false +// +// defer { +// if success { +// do { +// try FileManager.default.removeItem(at: imageFullPath) +// } catch { +// ELOG("Failed to delete image at path \(imageFullPath) \n \(error.localizedDescription)") +// } +// } +// } +// +// let gameFilename: String = imageFullPath.deletingPathExtension().lastPathComponent +// let gameExtension = imageFullPath.deletingPathExtension().pathExtension +// let database = RomDatabase.sharedInstance +// +// if gameExtension.isEmpty { +// ILOG("Trying to import artwork that didn't contain the extension of the system") +// let games = database.all(PVGame.self, filter: NSPredicate(format: "romPath CONTAINS[c] %@", argumentArray: [gameFilename])) +// +// if games.count == 1, let game = games.first { +// ILOG("File for image didn't have extension for system but we found a single match for image \(imageFullPath.lastPathComponent) to game \(game.title) on system \(game.systemIdentifier)") +// guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { +// return nil +// } +// +// do { +// try database.writeTransaction { +// game.customArtworkURL = hash +// } +// success = true +// ILOG("Set custom artwork of game \(game.title) from file \(imageFullPath.lastPathComponent)") +// } catch { +// ELOG("Couldn't update game \(game.title) with new artwork URL \(hash)") +// } +// +// return game +// } else { +// VLOG("Database search returned \(games.count) results") +// } +// } +// +// guard let systems: [PVSystem] = PVEmulatorConfiguration.systemsFromCache(forFileExtension: gameExtension), !systems.isEmpty else { +// ELOG("No system for extension \(gameExtension)") +// return nil +// } +// +// let cdBasedSystems = PVEmulatorConfiguration.cdBasedSystems +// let couldBelongToCDSystem = !Set(cdBasedSystems).isDisjoint(with: Set(systems)) +// +// if (couldBelongToCDSystem && PVEmulatorConfiguration.supportedCDFileExtensions.contains(gameExtension.lowercased())) || systems.count > 1 { +// guard let existingGames = findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: gameFilename) else { +// ELOG("System for extension \(gameExtension) is a CD system and {\(gameExtension)} not the right matching file type of cue or m3u") +// return nil +// } +// if existingGames.count == 1, let onlyMatch = existingGames.first { +// ILOG("We found a hit for artwork that could have been belonging to multiple games and only found one file that matched by systemid/filename. The winner is \(onlyMatch.title) for \(onlyMatch.systemIdentifier)") +// +// guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { +// ELOG("Couldn't move image, fail to set custom artwork") +// return nil +// } +// +// do { +// try database.writeTransaction { +// onlyMatch.customArtworkURL = hash +// } +// success = true +// } catch { +// ELOG("Couldn't update game \(onlyMatch.title) with new artwork URL") +// } +// return onlyMatch +// } else { +// ELOG("We got to the unlikely scenario where an extension is possibly a CD binary file, or belongs to a system, and had multiple games that matched the filename under more than one core.") +// return nil +// } +// } +// +// guard let system = systems.first else { +// ELOG("systems empty") +// return nil +// } +// +// var gamePartialPath: String = URL(fileURLWithPath: system.identifier, isDirectory: true).appendingPathComponent(gameFilename).deletingPathExtension().path +// if gamePartialPath.first == "/" { +// gamePartialPath.removeFirst() +// } +// +// if gamePartialPath.isEmpty { +// ELOG("Game path was empty") +// return nil +// } +// +// var games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), value: gamePartialPath) +// if games.isEmpty { +// games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), beginsWith: gamePartialPath) +// } +// +// guard !games.isEmpty else { +// ELOG("Couldn't find game for path \(gamePartialPath)") +// return nil +// } +// +// if games.count > 1 { +// WLOG("There were multiple matches for \(gamePartialPath)! #\(games.count). Going with first for now until we make better code to prompt user.") +// } +// +// let game = games.first! +// +// guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { +// ELOG("scaleAndMoveImageToCache failed") +// return nil +// } +// +// do { +// try database.writeTransaction { +// game.customArtworkURL = hash +// } +// success = true +// } catch { +// ELOG("Couldn't update game with new artwork URL") +// } +// +// return game +// } +// +// /// Scales and moves an image to the cache +// fileprivate class func scaleAndMoveImageToCache(imageFullPath: URL) -> String? { +// let coverArtFullData: Data +// do { +// coverArtFullData = try Data(contentsOf: imageFullPath, options: []) +// } catch { +// ELOG("Couldn't read data from image file \(imageFullPath.path)\n\(error.localizedDescription)") +// return nil +// } +// +//#if canImport(UIKit) +// guard let coverArtFullImage = UIImage(data: coverArtFullData) else { +// ELOG("Failed to create Image from data") +// return nil +// } +// guard let coverArtScaledImage = coverArtFullImage.scaledImage(withMaxResolution: Int(PVThumbnailMaxResolution)) else { +// ELOG("Failed to create scale image") +// return nil +// } +//#else +// guard let coverArtFullImage = NSImage(data: coverArtFullData) else { +// ELOG("Failed to create Image from data") +// return nil +// } +// let coverArtScaledImage = coverArtFullImage +//#endif +// +//#if canImport(UIKit) +// guard let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) else { +// ELOG("Failed to create data representation of scaled image") +// return nil +// } +//#else +// let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) +//#endif +// +// let hash: String = (coverArtScaledData as NSData).md5 +// +// do { +// let destinationURL = try PVMediaCache.writeData(toDisk: coverArtScaledData, withKey: hash) +// VLOG("Scaled and moved image from \(imageFullPath.path) to \(destinationURL.path)") +// } catch { +// ELOG("Failed to save artwork to cache: \(error.localizedDescription)") +// return nil +// } +// +// return hash +// } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift deleted file mode 100644 index 85001c5a3d..0000000000 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+ROMLookup.swift +++ /dev/null @@ -1,16 +0,0 @@ -// -// DatabaseQueryError.swift -// PVLibrary -// -// Created by Joseph Mattiello on 8/6/24. -// - -import PVLogging -import PVRealm -import PVMediaCache -import Systems - -// MARK: - ROM Lookup -public extension GameImporter { - -} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 4c09ac7678..9346f8e9b7 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -143,7 +143,8 @@ public final class GameImporter: GameImporting, ObservableObject { /// Singleton instance of GameImporter public static let shared: GameImporter = GameImporter(FileManager.default, GameImporterFileService(), - GameImporterDatabaseService()) + GameImporterDatabaseService(), + GameImporterSystemsService()) /// Instance of OpenVGDB for database operations var openVGDB = OpenVGDB.init() @@ -166,9 +167,6 @@ public final class GameImporter: GameImporting, ObservableObject { /// Map of system identifiers to their ROM paths public internal(set) var systemToPathMap = [String: URL]() - /// Map of ROM extensions to their corresponding system identifiers - public internal(set) var romExtensionToSystemsMap = [String: [String]]() - // MARK: - Queue public var importStatus: String = "" @@ -179,6 +177,7 @@ public final class GameImporter: GameImporting, ObservableObject { internal var gameImporterFileService:GameImporterFileServicing internal var gameImporterDatabaseService:GameImporterDatabaseServicing + internal var gameImporterSystemsService:GameImporterSystemsServicing // MARK: - Paths @@ -211,17 +210,20 @@ public final class GameImporter: GameImporting, ObservableObject { /// Initializes the GameImporter internal init(_ fm: FileManager, _ fileService:GameImporterFileServicing, - _ databaseService:GameImporterDatabaseServicing) { + _ databaseService:GameImporterDatabaseServicing, + _ systemsService:GameImporterSystemsServicing) { gameImporterFileService = fileService gameImporterDatabaseService = databaseService + gameImporterSystemsService = systemsService //create defaults createDefaultDirectories(fm: fm) - //set the romsPath propery of the db service, since it needs access + //set service dependencies gameImporterDatabaseService.setRomsPath(url: romsPath) gameImporterDatabaseService.setOpenVGDB(openVGDB) + gameImporterSystemsService.setOpenVGDB(openVGDB) } /// Creates default directories @@ -284,14 +286,14 @@ public final class GameImporter: GameImporting, ObservableObject { Task.detached { ILOG("RealmCollection changed state to .initial") self.systemToPathMap = await updateSystemToPathMap() - self.romExtensionToSystemsMap = updateromExtensionToSystemsMap() + self.gameImporterSystemsService.setExtensionsToSystemMapping(updateromExtensionToSystemsMap()) self.initialized.leave() } case .update: Task.detached { ILOG("RealmCollection changed state to .update") self.systemToPathMap = await updateSystemToPathMap() - self.romExtensionToSystemsMap = updateromExtensionToSystemsMap() + self.gameImporterSystemsService.setExtensionsToSystemMapping(updateromExtensionToSystemsMap()) } case let .error(error): ELOG("RealmCollection changed state to .error") @@ -572,7 +574,7 @@ public final class GameImporter: GameImporting, ObservableObject { item.fileType = try determineImportType(item) //get valid systems that this object might support - guard let systems = try? await determineSystems(for: item), !systems.isEmpty else { + guard let systems = try? await gameImporterSystemsService.determineSystems(for: item), !systems.isEmpty else { //this is actually an import error item.status = .failure throw GameImporterError.noSystemMatched diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 9ec519d4b1..3d9fc66473 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -49,11 +49,11 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } internal func importGameIntoDatabase(queueItem: ImportQueueItem) async throws { - guard let targetSystem = queueItem.systems.first else { + guard let targetSystem = queueItem.targetSystem() else { throw GameImporterError.systemNotDetermined } - DLOG("Attempting to import game: \(queueItem.url.lastPathComponent) for system: \(targetSystem.name)") + DLOG("Attempting to import game: \(queueItem.destinationUrl?.lastPathComponent) for system: \(targetSystem.name)") let filename = queueItem.url.lastPathComponent let partialPath = (targetSystem.identifier as NSString).appendingPathComponent(filename) @@ -82,7 +82,11 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") - let file = PVFile(withURL: queueItem.url) + guard let destUrl = queueItem.destinationUrl else { + //how did we get here, throw? + return + } + let file = PVFile(withURL: queueItem.destinationUrl!) let game = PVGame(withFile: file, system: system) game.romPath = partialPath game.title = title @@ -116,20 +120,6 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } DLOG("Calculated MD5: \(md5)") -// // Register import with coordinator -// guard await importCoordinator.checkAndRegisterImport(md5: md5) else { -// DLOG("Import already in progress for MD5: \(md5)") -// throw GameImporterError.romAlreadyExistsInDatabase -// } -// DLOG("Registered import with coordinator for MD5: \(md5)") - -// defer { -// Task { -// await importCoordinator.completeImport(md5: md5) -// DLOG("Completed import coordination for MD5: \(md5)") -// } -// } - game.relatedFiles.append(objectsIn: relatedPVFiles) game.md5Hash = md5 try await finishUpdateOrImport(ofGame: game) @@ -148,25 +138,10 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if RomDatabase.gamesCache[game.romPath] != nil { throw GameImporterError.romAlreadyExistsInDatabase } - var modified = false var game:PVGame = game if game.requiresSync { -// if importStartedHandler != nil { -// let fullpath = PVEmulatorConfiguration.path(forGame: game) -// Task { @MainActor in -// self.importStartedHandler?(fullpath.path) -// } -// } - game = lookupInfo(for: game, overwrite: true) - modified = true - } - let wasModified = modified -// if finishedImportHandler != nil { -// let md5: String = game.md5Hash -// // Task { @MainActor in -// self.finishedImportHandler?(md5, wasModified) -// // } -// } + game = getUpdatedGameInfo(for: game, forceRefresh: true) + } if game.originalArtworkFile == nil { game = await getArtwork(forGame: game) } @@ -225,9 +200,101 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { //MARK: Utility + private func updateGameFields(_ game: PVGame, gameDBRecordInfo: [String : Any], forceRefresh: Bool) -> PVGame { + //we found the record for the target PVGame + + /* Optional results + gameTitle + boxImageURL + region + gameDescription + boxBackURL + developer + publisher + year + genres [comma array string] + referenceURL + releaseID + regionID + systemShortName + serial + */ + + //this section updates the various fields in the return record. + if let title = gameDBRecordInfo["gameTitle"] as? String, !title.isEmpty, forceRefresh || game.title.isEmpty { + // Remove just (Disc 1) from the title. Discs with other numbers will retain their names + let revisedTitle = title.replacingOccurrences(of: "\\ \\(Disc 1\\)", with: "", options: .regularExpression) + game.title = revisedTitle + } + + if let boxImageURL = gameDBRecordInfo["boxImageURL"] as? String, !boxImageURL.isEmpty, forceRefresh || game.originalArtworkURL.isEmpty { + game.originalArtworkURL = boxImageURL + } + + if let regionName = gameDBRecordInfo["region"] as? String, !regionName.isEmpty, forceRefresh || game.regionName == nil { + game.regionName = regionName + } + + if let regionID = gameDBRecordInfo["regionID"] as? Int, forceRefresh || game.regionID.value == nil { + game.regionID.value = regionID + } + + if let gameDescription = gameDBRecordInfo["gameDescription"] as? String, !gameDescription.isEmpty, forceRefresh || game.gameDescription == nil { + let options = [NSAttributedString.DocumentReadingOptionKey.documentType: NSAttributedString.DocumentType.html] + if let data = gameDescription.data(using: .isoLatin1) { + do { + let htmlDecodedGameDescription = try NSMutableAttributedString(data: data, options: options, documentAttributes: nil) + game.gameDescription = htmlDecodedGameDescription.string.replacingOccurrences(of: "(\\.|\\!|\\?)([A-Z][A-Za-z\\s]{2,})", with: "$1\n\n$2", options: .regularExpression) + } catch { + ELOG("\(error.localizedDescription)") + } + } + } + + if let boxBackURL = gameDBRecordInfo["boxBackURL"] as? String, !boxBackURL.isEmpty, forceRefresh || game.boxBackArtworkURL == nil { + game.boxBackArtworkURL = boxBackURL + } + + if let developer = gameDBRecordInfo["developer"] as? String, !developer.isEmpty, forceRefresh || game.developer == nil { + game.developer = developer + } + + if let publisher = gameDBRecordInfo["publisher"] as? String, !publisher.isEmpty, forceRefresh || game.publisher == nil { + game.publisher = publisher + } + + if let genres = gameDBRecordInfo["genres"] as? String, !genres.isEmpty, forceRefresh || game.genres == nil { + game.genres = genres + } + + if let releaseDate = gameDBRecordInfo["releaseDate"] as? String, !releaseDate.isEmpty, forceRefresh || game.publishDate == nil { + game.publishDate = releaseDate + } + + if let referenceURL = gameDBRecordInfo["referenceURL"] as? String, !referenceURL.isEmpty, forceRefresh || game.referenceURL == nil { + game.referenceURL = referenceURL + } + + if let releaseID = gameDBRecordInfo["releaseID"] as? NSNumber, !releaseID.stringValue.isEmpty, forceRefresh || game.releaseID == nil { + game.releaseID = releaseID.stringValue + } + + if let systemShortName = gameDBRecordInfo["systemShortName"] as? String, !systemShortName.isEmpty, forceRefresh || game.systemShortName == nil { + game.systemShortName = systemShortName + } + + if let romSerial = gameDBRecordInfo["serial"] as? String, !romSerial.isEmpty, forceRefresh || game.romSerial == nil { + game.romSerial = romSerial + } + + return game + } + @discardableResult - func lookupInfo(for game: PVGame, overwrite: Bool = true) -> PVGame { + func getUpdatedGameInfo(for game: PVGame, forceRefresh: Bool = true) -> PVGame { game.requiresSync = false + + //step 1 - calculate md5 hash if needed if game.md5Hash.isEmpty { if let romFullPath = romsPath?.appendingPathComponent(game.romPath).path { if let md5Hash = calculateMD5(forGame: game) { @@ -239,6 +306,8 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { NSLog("Game md5 has was empty") return game } + + //step 2 - check art cache or database id for info based on md5 hash var resultsMaybe: [[String: Any]]? do { if let result = RomDatabase.getArtCache(game.md5Hash.uppercased(), systemIdentifier:game.systemIdentifier) { @@ -249,6 +318,8 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } catch { ELOG("\(error.localizedDescription)") } + + //step 3 - still didn't find any candidate results, check by file name if resultsMaybe == nil || resultsMaybe!.isEmpty { //PVEmulatorConfiguration.supportedROMFileExtensions.contains(game.file.url.pathExtension.lowercased()) { let fileName: String = game.file.url.lastPathComponent // Remove any extraneous stuff in the rom name such as (U), (J), [T+Eng] etc @@ -270,6 +341,8 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { ELOG("\(error.localizedDescription)") } } + + //still got nothing, so just return the partial record we got. guard let results = resultsMaybe, !results.isEmpty else { // the file maybe exists but was wiped from DB, // try to re-import and rescan if can @@ -289,6 +362,8 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { */ return game } + + //this block seems to pick the USA record from the DB if there are multiple options? var chosenResultMaybe: [String: Any]? = // Search by region id results.first { (dict) -> Bool in @@ -308,100 +383,14 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } chosenResultMaybe = results.first } - //write at the end of fininshOrUpdateImport - //autoreleasepool { - // do { + game.requiresSync = false guard let chosenResult = chosenResultMaybe else { NSLog("Unable to find ROM \(game.romPath) in OpenVGDB") return game } - /* Optional results - gameTitle - boxImageURL - region - gameDescription - boxBackURL - developer - publisher - year - genres [comma array string] - referenceURL - releaseID - regionID - systemShortName - serial - */ - if let title = chosenResult["gameTitle"] as? String, !title.isEmpty, overwrite || game.title.isEmpty { - // Remove just (Disc 1) from the title. Discs with other numbers will retain their names - let revisedTitle = title.replacingOccurrences(of: "\\ \\(Disc 1\\)", with: "", options: .regularExpression) - game.title = revisedTitle - } - - if let boxImageURL = chosenResult["boxImageURL"] as? String, !boxImageURL.isEmpty, overwrite || game.originalArtworkURL.isEmpty { - game.originalArtworkURL = boxImageURL - } - - if let regionName = chosenResult["region"] as? String, !regionName.isEmpty, overwrite || game.regionName == nil { - game.regionName = regionName - } - - if let regionID = chosenResult["regionID"] as? Int, overwrite || game.regionID.value == nil { - game.regionID.value = regionID - } - - if let gameDescription = chosenResult["gameDescription"] as? String, !gameDescription.isEmpty, overwrite || game.gameDescription == nil { - let options = [NSAttributedString.DocumentReadingOptionKey.documentType: NSAttributedString.DocumentType.html] - if let data = gameDescription.data(using: .isoLatin1) { - do { - let htmlDecodedGameDescription = try NSMutableAttributedString(data: data, options: options, documentAttributes: nil) - game.gameDescription = htmlDecodedGameDescription.string.replacingOccurrences(of: "(\\.|\\!|\\?)([A-Z][A-Za-z\\s]{2,})", with: "$1\n\n$2", options: .regularExpression) - } catch { - ELOG("\(error.localizedDescription)") - } - } - } - - if let boxBackURL = chosenResult["boxBackURL"] as? String, !boxBackURL.isEmpty, overwrite || game.boxBackArtworkURL == nil { - game.boxBackArtworkURL = boxBackURL - } - - if let developer = chosenResult["developer"] as? String, !developer.isEmpty, overwrite || game.developer == nil { - game.developer = developer - } - - if let publisher = chosenResult["publisher"] as? String, !publisher.isEmpty, overwrite || game.publisher == nil { - game.publisher = publisher - } - - if let genres = chosenResult["genres"] as? String, !genres.isEmpty, overwrite || game.genres == nil { - game.genres = genres - } - - if let releaseDate = chosenResult["releaseDate"] as? String, !releaseDate.isEmpty, overwrite || game.publishDate == nil { - game.publishDate = releaseDate - } - - if let referenceURL = chosenResult["referenceURL"] as? String, !referenceURL.isEmpty, overwrite || game.referenceURL == nil { - game.referenceURL = referenceURL - } - - if let releaseID = chosenResult["releaseID"] as? NSNumber, !releaseID.stringValue.isEmpty, overwrite || game.releaseID == nil { - game.releaseID = releaseID.stringValue - } - - if let systemShortName = chosenResult["systemShortName"] as? String, !systemShortName.isEmpty, overwrite || game.systemShortName == nil { - game.systemShortName = systemShortName - } - - if let romSerial = chosenResult["serial"] as? String, !romSerial.isEmpty, overwrite || game.romSerial == nil { - game.romSerial = romSerial - } - // } catch { - // ELOG("Failed to update game \(game.title) : \(error.localizedDescription)") - // } - //} - return game + + return updateGameFields(game, gameDBRecordInfo:chosenResult, forceRefresh:forceRefresh) } func releaseID(forCRCs crcs: Set) -> String? { @@ -448,6 +437,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { /// Saves a game to the database func saveGame(_ game:PVGame) { do { + //TODO: this might crash if not on main thread - validate let database = RomDatabase.sharedInstance try database.writeTransaction { database.realm.create(PVGame.self, value:game, update:.modified) @@ -465,6 +455,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { //this seems to be spread in many places, not sure why. it might be doable to put this in the queue item, but for now, trying to consolidate. //I have no history or explanation for why we need the 16 offset for SNES/NES + // the legacy code was actually inconsistently applied, so there's a good chance this causes some bugs (or fixes some) if game.systemIdentifier == SystemIdentifier.SNES.rawValue { offset = SystemIdentifier.SNES.offset } else if game.systemIdentifier == SystemIdentifier.NES.rawValue { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index dc0f6d743c..0075f956ad 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -28,11 +28,8 @@ class GameImporterFileService : GameImporterFileServicing { case .artwork: //TODO: implement me return - case .game: - _ = try await handleROM(queueItem) - return - case .cdRom: - _ = try await handleCDROMItem(queueItem) + case .game,.cdRom: + _ = try await processQueueItem(queueItem) case .unknown: throw GameImporterError.unsupportedFile } @@ -64,14 +61,15 @@ class GameImporterFileService : GameImporterFileServicing { } } } - //MARK: - Normal ROMs + //MARK: - Normal ROMs and CDROMs - /// Moves a ROM to the appropriate subfolder - internal func handleROM(_ queueItem: ImportQueueItem) async throws { - guard queueItem.fileType == .game else { + /// Moves an ImportQueueItem to the appropriate subfolder + internal func processQueueItem(_ queueItem: ImportQueueItem) async throws { + guard queueItem.fileType == .game || queueItem.fileType == .cdRom else { throw GameImporterError.unsupportedFile } + //this might not be needed... guard !queueItem.systems.isEmpty else { throw GameImporterError.noSystemMatched } @@ -84,41 +82,35 @@ class GameImporterFileService : GameImporterFileServicing { let destinationFolder = targetSystem.romsDirectory let destinationPath = destinationFolder.appendingPathComponent(fileName) - queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) - } - - // MARK: - CDROM - - /// Ensures a BIOS file is copied to appropriate file destinations - /// Returns the array of PVSystems that this BIOS file applies to - private func handleCDROMItem(_ queueItem: ImportQueueItem) async throws { - guard queueItem.fileType == .cdRom else { - throw GameImporterError.unsupportedCDROMFile - } - - guard !queueItem.systems.isEmpty else { - throw GameImporterError.unsupportedCDROMFile - } - - guard let targetSystem = queueItem.targetSystem() else { - throw GameImporterError.systemNotDetermined - } - - let fileName = queueItem.url.lastPathComponent - let destinationFolder = targetSystem.romsDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) - - //TODO: check and process children - do { queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) + try await moveChildImports(forQueueItem: queueItem, to: destinationFolder) } catch { - throw GameImporterError.failedToMoveCDROM(error) + throw GameImporterError.failedToMoveROM(error) } } // MARK: - Utility + internal func moveChildImports(forQueueItem queueItem:ImportQueueItem, to destinationFolder:URL) async throws { + guard !queueItem.childQueueItems.isEmpty else { + return + } + + for childQueueItem in queueItem.childQueueItems { + let fileName = childQueueItem.url.lastPathComponent + let destinationPath = destinationFolder.appendingPathComponent(fileName) + + do { + childQueueItem.destinationUrl = try await moveFile(childQueueItem.url, to: destinationPath) + //call recursively to keep moving child items to the target directory as a unit + try await moveChildImports(forQueueItem: childQueueItem, to: destinationFolder) + } catch { + throw GameImporterError.failedToMoveCDROM(error) + } + } + } + /// Moves a file to the conflicts directory internal func moveToConflictsFolder(_ queueItem: ImportQueueItem, conflictsPath: URL) async throws { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift similarity index 93% rename from PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift rename to PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index e8b0af4a03..6dddb235ba 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Systems.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -1,8 +1,8 @@ // -// GameImporter+Systems.swift +// File.swift // PVLibrary // -// Created by David Proskin on 11/3/24. +// Created by David Proskin on 11/7/24. // import Foundation @@ -21,7 +21,28 @@ import PVRealm import Perception import SwiftUI -extension GameImporter { +protocol GameImporterSystemsServicing { + func setOpenVGDB(_ vgdb: OpenVGDB) + func setExtensionsToSystemMapping(_ mapping: [String: [String]]) + func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] +} + +class GameImporterSystemsService : GameImporterSystemsServicing { + var openVGDB: OpenVGDB? + /// Map of ROM extensions to their corresponding system identifiers + var romExtensionToSystemsMap = [String: [String]]() + + init() { + + } + + func setOpenVGDB(_ vgdb: OpenVGDB) { + openVGDB = vgdb + } + + func setExtensionsToSystemMapping(_ mapping: [String: [String]]) { + romExtensionToSystemsMap = mapping + } internal func matchSystemByPartialName(_ fileName: String, possibleSystems: [PVSystem]) -> PVSystem? { let cleanedName = fileName.lowercased() @@ -71,7 +92,7 @@ extension GameImporter { // If no match found, try querying the OpenVGDB //TODO: fix me do { - if let results = try openVGDB.searchDatabase(usingFilename: fileName), + if let results = try openVGDB?.searchDatabase(usingFilename: fileName), let firstResult = results.first, let systemID = firstResult["systemID"] as? Int, let system = PVEmulatorConfiguration.system(forDatabaseID: systemID) { @@ -115,7 +136,7 @@ extension GameImporter { let cleanedFileName = cleanFileName(lowercasedFileName) // Search the database using the cleaned filename - if let results = try openVGDB.searchDatabase(usingFilename: cleanedFileName, systemID: system.openvgDatabaseID) { + if let results = try openVGDB?.searchDatabase(usingFilename: cleanedFileName, systemID: system.openvgDatabaseID) { // Check if we have any results if !results.isEmpty { // Optionally, you can add more strict matching here @@ -264,7 +285,7 @@ extension GameImporter { for system in possibleSystems { do { - if let results = try openVGDB.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), + if let results = try openVGDB?.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), !results.isEmpty { ILOG("System determined by filename match in OpenVGDB: \(system.name)") return system @@ -277,7 +298,7 @@ extension GameImporter { // If we couldn't determine the system, try a more detailed search if let fileMD5 = queueItem.md5?.uppercased(), !fileMD5.isEmpty { do { - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: fileMD5), + if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: fileMD5), let firstResult = results.first, let systemID = firstResult["systemID"] as? Int, let system = possibleSystems.first(where: { $0.openvgDatabaseID == systemID }) { @@ -315,7 +336,7 @@ extension GameImporter { } /// Determines the system for a given candidate file - internal func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] { + public func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] { guard let md5 = queueItem.md5?.uppercased() else { throw GameImporterError.couldNotCalculateMD5 } @@ -349,7 +370,7 @@ extension GameImporter { } // Try to find system by MD5 using OpenVGDB - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), + if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), let firstResult = results.first, let systemID = firstResult["systemID"] as? NSNumber { @@ -427,7 +448,7 @@ extension GameImporter { let fileName: String = queueItem.url.lastPathComponent do { - if let databaseID = try openVGDB.system(forRomMD5: md5, or: fileName), + if let databaseID = try openVGDB?.system(forRomMD5: md5, or: fileName), let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { return systemID } else { @@ -448,7 +469,7 @@ extension GameImporter { DLOG("Attempting MD5 lookup for: \(md5)") // Try to find system by MD5 using OpenVGDB - if let results = try openVGDB.searchDatabase(usingKey: "romHashMD5", value: md5), + if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), let firstResult = results.first, let systemID = firstResult["systemID"] as? NSNumber, let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { @@ -503,3 +524,4 @@ extension GameImporter { return romExtensionToSystemsMap[fileExtension] } } + From 21cef9caea9acc68f5bf3fedf910988d347a4ce5 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 13:13:50 -0500 Subject: [PATCH 07/30] Added artwork importer code to try and handle artwork properly. additional code and warning clean up --- .../Importer/Models/GameImporterError.swift | 1 + .../GameImporter/ArtworkImporter.swift | 214 ++++++++++++++++++ .../GameImporter/GameImporter+Artwork.swift | 193 ---------------- .../Services/GameImporter/GameImporter.swift | 27 ++- .../GameImporterDatabaseService.swift | 21 +- .../GameImporterFileService.swift | 2 +- .../GameImporterSystemsService.swift | 5 +- 7 files changed, 256 insertions(+), 207 deletions(-) create mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ArtworkImporter.swift delete mode 100644 PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift index 2f61d56a8b..84e9a829cf 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift @@ -16,4 +16,5 @@ public enum GameImporterError: Error, Sendable { case unsupportedFile case noBIOSMatchForBIOSFileType case unsupportedCDROMFile + case incorrectDestinationURL } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ArtworkImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ArtworkImporter.swift new file mode 100644 index 0000000000..7bebc00104 --- /dev/null +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/ArtworkImporter.swift @@ -0,0 +1,214 @@ +// +// ArtworkImporter.swift +// PVLibrary +// +// Created by David Proskin on 11/3/24. +// + +protocol ArtworkImporting { + func setSystemsService(_ systemsService:GameImporterSystemsServicing) + func importArtworkItem(_ queueItem: ImportQueueItem) async -> PVGame? +} + +class ArtworkImporter : ArtworkImporting { + + private var gameImporterSystemsService:GameImporterSystemsServicing? + + init() { + + } + + func setSystemsService(_ systemsService:GameImporterSystemsServicing) { + gameImporterSystemsService = systemsService + } + + func importArtworkItem(_ queueItem: ImportQueueItem) async -> PVGame? { + guard queueItem.fileType == .artwork else { + return nil + } + + var isDirectory: ObjCBool = false + let fileExists = FileManager.default.fileExists(atPath: queueItem.url.path, isDirectory: &isDirectory) + if !fileExists || isDirectory.boolValue { + WLOG("File doesn't exist or is directory at \(queueItem.url)") + return nil + } + + var success = false + + defer { + if success { + do { + //clean up... + try FileManager.default.removeItem(at: queueItem.url) + } catch { + ELOG("Failed to delete image at path \(queueItem.url) \n \(error.localizedDescription)") + } + } + } + + //I think what this does is rely on a file being something like game.sfc.jpg or something? + let gameFilename: String = queueItem.url.deletingPathExtension().lastPathComponent + let gameExtension = queueItem.url.deletingPathExtension().pathExtension + let database = RomDatabase.sharedInstance + + if gameExtension.isEmpty { + ILOG("Trying to import artwork that didn't contain the extension of the system") + let games = database.all(PVGame.self, filter: NSPredicate(format: "romPath CONTAINS[c] %@", argumentArray: [gameFilename])) + + if games.count == 1, let game = games.first { + ILOG("File for image didn't have extension for system but we found a single match for image \(queueItem.url.lastPathComponent) to game \(game.title) on system \(game.systemIdentifier)") + guard let hash = scaleAndMoveImageToCache(imageFullPath: queueItem.url) else { + return nil + } + + do { + try database.writeTransaction { + game.customArtworkURL = hash + } + success = true + ILOG("Set custom artwork of game \(game.title) from file \(queueItem.url.lastPathComponent)") + } catch { + ELOG("Couldn't update game \(game.title) with new artwork URL \(hash)") + } + + return game + } else { + VLOG("Database search returned \(games.count) results") + //TODO: consider either using it for all or asking the user what to do - for now this is an error + } + } + + guard let systems: [PVSystem] = PVEmulatorConfiguration.systemsFromCache(forFileExtension: gameExtension), !systems.isEmpty else { + ELOG("No system for extension \(gameExtension)") + return nil + } + + let cdBasedSystems = PVEmulatorConfiguration.cdBasedSystems + let couldBelongToCDSystem = !Set(cdBasedSystems).isDisjoint(with: Set(systems)) + + if (couldBelongToCDSystem && PVEmulatorConfiguration.supportedCDFileExtensions.contains(gameExtension.lowercased())) || systems.count > 1 { + guard let existingGames = gameImporterSystemsService?.findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: gameFilename) else { + ELOG("System for extension \(gameExtension) is a CD system and {\(gameExtension)} not the right matching file type of cue or m3u") + return nil + } + if existingGames.count == 1, let onlyMatch = existingGames.first { + ILOG("We found a hit for artwork that could have been belonging to multiple games and only found one file that matched by systemid/filename. The winner is \(onlyMatch.title) for \(onlyMatch.systemIdentifier)") + + guard let hash = scaleAndMoveImageToCache(imageFullPath: queueItem.url) else { + ELOG("Couldn't move image, fail to set custom artwork") + return nil + } + + do { + try database.writeTransaction { + onlyMatch.customArtworkURL = hash + } + success = true + } catch { + ELOG("Couldn't update game \(onlyMatch.title) with new artwork URL") + } + return onlyMatch + } else { + ELOG("We got to the unlikely scenario where an extension is possibly a CD binary file, or belongs to a system, and had multiple games that matched the filename under more than one core.") + return nil + } + } + + guard let system = systems.first else { + ELOG("systems empty") + return nil + } + + var gamePartialPath: String = URL(fileURLWithPath: system.identifier, isDirectory: true).appendingPathComponent(gameFilename).deletingPathExtension().path + if gamePartialPath.first == "/" { + gamePartialPath.removeFirst() + } + + if gamePartialPath.isEmpty { + ELOG("Game path was empty") + return nil + } + + var games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), value: gamePartialPath) + if games.isEmpty { + games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), beginsWith: gamePartialPath) + } + + guard !games.isEmpty else { + ELOG("Couldn't find game for path \(gamePartialPath)") + return nil + } + + if games.count > 1 { + WLOG("There were multiple matches for \(gamePartialPath)! #\(games.count). Going with first for now until we make better code to prompt user.") + } + + let game = games.first! + + guard let hash = scaleAndMoveImageToCache(imageFullPath: queueItem.url) else { + ELOG("scaleAndMoveImageToCache failed") + return nil + } + + do { + try database.writeTransaction { + game.customArtworkURL = hash + } + success = true + } catch { + ELOG("Couldn't update game with new artwork URL") + } + + return game + } + + /// Scales and moves an image to the cache + private func scaleAndMoveImageToCache(imageFullPath: URL) -> String? { + let coverArtFullData: Data + do { + coverArtFullData = try Data(contentsOf: imageFullPath, options: []) + } catch { + ELOG("Couldn't read data from image file \(imageFullPath.path)\n\(error.localizedDescription)") + return nil + } + +#if canImport(UIKit) + guard let coverArtFullImage = UIImage(data: coverArtFullData) else { + ELOG("Failed to create Image from data") + return nil + } + guard let coverArtScaledImage = coverArtFullImage.scaledImage(withMaxResolution: Int(PVThumbnailMaxResolution)) else { + ELOG("Failed to create scale image") + return nil + } +#else + guard let coverArtFullImage = NSImage(data: coverArtFullData) else { + ELOG("Failed to create Image from data") + return nil + } + let coverArtScaledImage = coverArtFullImage +#endif + +#if canImport(UIKit) + guard let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) else { + ELOG("Failed to create data representation of scaled image") + return nil + } +#else + let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) +#endif + + let hash: String = (coverArtScaledData as NSData).md5 + + do { + let destinationURL = try PVMediaCache.writeData(toDisk: coverArtScaledData, withKey: hash) + VLOG("Scaled and moved image from \(imageFullPath.path) to \(destinationURL.path)") + } catch { + ELOG("Failed to save artwork to cache: \(error.localizedDescription)") + return nil + } + + return hash + } +} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift deleted file mode 100644 index 029240962b..0000000000 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter+Artwork.swift +++ /dev/null @@ -1,193 +0,0 @@ -// -// func.swift -// PVLibrary -// -// Created by David Proskin on 11/3/24. -// - - -public extension GameImporter { - /// Imports artwork from a given path -// class func importArtwork(fromPath imageFullPath: URL) async -> PVGame? { -// var isDirectory: ObjCBool = false -// let fileExists = FileManager.default.fileExists(atPath: imageFullPath.path, isDirectory: &isDirectory) -// if !fileExists || isDirectory.boolValue { -// WLOG("File doesn't exist or is directory at \(imageFullPath)") -// return nil -// } -// -// var success = false -// -// defer { -// if success { -// do { -// try FileManager.default.removeItem(at: imageFullPath) -// } catch { -// ELOG("Failed to delete image at path \(imageFullPath) \n \(error.localizedDescription)") -// } -// } -// } -// -// let gameFilename: String = imageFullPath.deletingPathExtension().lastPathComponent -// let gameExtension = imageFullPath.deletingPathExtension().pathExtension -// let database = RomDatabase.sharedInstance -// -// if gameExtension.isEmpty { -// ILOG("Trying to import artwork that didn't contain the extension of the system") -// let games = database.all(PVGame.self, filter: NSPredicate(format: "romPath CONTAINS[c] %@", argumentArray: [gameFilename])) -// -// if games.count == 1, let game = games.first { -// ILOG("File for image didn't have extension for system but we found a single match for image \(imageFullPath.lastPathComponent) to game \(game.title) on system \(game.systemIdentifier)") -// guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { -// return nil -// } -// -// do { -// try database.writeTransaction { -// game.customArtworkURL = hash -// } -// success = true -// ILOG("Set custom artwork of game \(game.title) from file \(imageFullPath.lastPathComponent)") -// } catch { -// ELOG("Couldn't update game \(game.title) with new artwork URL \(hash)") -// } -// -// return game -// } else { -// VLOG("Database search returned \(games.count) results") -// } -// } -// -// guard let systems: [PVSystem] = PVEmulatorConfiguration.systemsFromCache(forFileExtension: gameExtension), !systems.isEmpty else { -// ELOG("No system for extension \(gameExtension)") -// return nil -// } -// -// let cdBasedSystems = PVEmulatorConfiguration.cdBasedSystems -// let couldBelongToCDSystem = !Set(cdBasedSystems).isDisjoint(with: Set(systems)) -// -// if (couldBelongToCDSystem && PVEmulatorConfiguration.supportedCDFileExtensions.contains(gameExtension.lowercased())) || systems.count > 1 { -// guard let existingGames = findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(systems, romFilename: gameFilename) else { -// ELOG("System for extension \(gameExtension) is a CD system and {\(gameExtension)} not the right matching file type of cue or m3u") -// return nil -// } -// if existingGames.count == 1, let onlyMatch = existingGames.first { -// ILOG("We found a hit for artwork that could have been belonging to multiple games and only found one file that matched by systemid/filename. The winner is \(onlyMatch.title) for \(onlyMatch.systemIdentifier)") -// -// guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { -// ELOG("Couldn't move image, fail to set custom artwork") -// return nil -// } -// -// do { -// try database.writeTransaction { -// onlyMatch.customArtworkURL = hash -// } -// success = true -// } catch { -// ELOG("Couldn't update game \(onlyMatch.title) with new artwork URL") -// } -// return onlyMatch -// } else { -// ELOG("We got to the unlikely scenario where an extension is possibly a CD binary file, or belongs to a system, and had multiple games that matched the filename under more than one core.") -// return nil -// } -// } -// -// guard let system = systems.first else { -// ELOG("systems empty") -// return nil -// } -// -// var gamePartialPath: String = URL(fileURLWithPath: system.identifier, isDirectory: true).appendingPathComponent(gameFilename).deletingPathExtension().path -// if gamePartialPath.first == "/" { -// gamePartialPath.removeFirst() -// } -// -// if gamePartialPath.isEmpty { -// ELOG("Game path was empty") -// return nil -// } -// -// var games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), value: gamePartialPath) -// if games.isEmpty { -// games = database.all(PVGame.self, where: #keyPath(PVGame.romPath), beginsWith: gamePartialPath) -// } -// -// guard !games.isEmpty else { -// ELOG("Couldn't find game for path \(gamePartialPath)") -// return nil -// } -// -// if games.count > 1 { -// WLOG("There were multiple matches for \(gamePartialPath)! #\(games.count). Going with first for now until we make better code to prompt user.") -// } -// -// let game = games.first! -// -// guard let hash = scaleAndMoveImageToCache(imageFullPath: imageFullPath) else { -// ELOG("scaleAndMoveImageToCache failed") -// return nil -// } -// -// do { -// try database.writeTransaction { -// game.customArtworkURL = hash -// } -// success = true -// } catch { -// ELOG("Couldn't update game with new artwork URL") -// } -// -// return game -// } -// -// /// Scales and moves an image to the cache -// fileprivate class func scaleAndMoveImageToCache(imageFullPath: URL) -> String? { -// let coverArtFullData: Data -// do { -// coverArtFullData = try Data(contentsOf: imageFullPath, options: []) -// } catch { -// ELOG("Couldn't read data from image file \(imageFullPath.path)\n\(error.localizedDescription)") -// return nil -// } -// -//#if canImport(UIKit) -// guard let coverArtFullImage = UIImage(data: coverArtFullData) else { -// ELOG("Failed to create Image from data") -// return nil -// } -// guard let coverArtScaledImage = coverArtFullImage.scaledImage(withMaxResolution: Int(PVThumbnailMaxResolution)) else { -// ELOG("Failed to create scale image") -// return nil -// } -//#else -// guard let coverArtFullImage = NSImage(data: coverArtFullData) else { -// ELOG("Failed to create Image from data") -// return nil -// } -// let coverArtScaledImage = coverArtFullImage -//#endif -// -//#if canImport(UIKit) -// guard let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) else { -// ELOG("Failed to create data representation of scaled image") -// return nil -// } -//#else -// let coverArtScaledData = coverArtScaledImage.jpegData(compressionQuality: 0.85) -//#endif -// -// let hash: String = (coverArtScaledData as NSData).md5 -// -// do { -// let destinationURL = try PVMediaCache.writeData(toDisk: coverArtScaledData, withKey: hash) -// VLOG("Scaled and moved image from \(imageFullPath.path) to \(destinationURL.path)") -// } catch { -// ELOG("Failed to save artwork to cache: \(error.localizedDescription)") -// return nil -// } -// -// return hash -// } -} diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 9346f8e9b7..8f1916ed51 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -144,7 +144,8 @@ public final class GameImporter: GameImporting, ObservableObject { public static let shared: GameImporter = GameImporter(FileManager.default, GameImporterFileService(), GameImporterDatabaseService(), - GameImporterSystemsService()) + GameImporterSystemsService(), + ArtworkImporter()) /// Instance of OpenVGDB for database operations var openVGDB = OpenVGDB.init() @@ -178,6 +179,7 @@ public final class GameImporter: GameImporting, ObservableObject { internal var gameImporterFileService:GameImporterFileServicing internal var gameImporterDatabaseService:GameImporterDatabaseServicing internal var gameImporterSystemsService:GameImporterSystemsServicing + internal var gameImporterArtworkImporter:ArtworkImporting // MARK: - Paths @@ -211,10 +213,12 @@ public final class GameImporter: GameImporting, ObservableObject { internal init(_ fm: FileManager, _ fileService:GameImporterFileServicing, _ databaseService:GameImporterDatabaseServicing, - _ systemsService:GameImporterSystemsServicing) { + _ systemsService:GameImporterSystemsServicing, + _ artworkImporter:ArtworkImporting) { gameImporterFileService = fileService gameImporterDatabaseService = databaseService gameImporterSystemsService = systemsService + gameImporterArtworkImporter = artworkImporter //create defaults createDefaultDirectories(fm: fm) @@ -224,6 +228,8 @@ public final class GameImporter: GameImporting, ObservableObject { gameImporterDatabaseService.setOpenVGDB(openVGDB) gameImporterSystemsService.setOpenVGDB(openVGDB) + + gameImporterArtworkImporter.setSystemsService(gameImporterSystemsService) } /// Creates default directories @@ -570,9 +576,19 @@ public final class GameImporter: GameImporting, ObservableObject { private func performImport(for item: ImportQueueItem) async throws { - //ideally this wouldn't be needed here + //ideally this wouldn't be needed here because we'd have done it elsewhere item.fileType = try determineImportType(item) + if item.fileType == .artwork { + //TODO: what do i do with the PVGame result here? + if let _ = await gameImporterArtworkImporter.importArtworkItem(item) { + item.status = .success + } else { + item.status = .failure + } + return + } + //get valid systems that this object might support guard let systems = try? await gameImporterSystemsService.determineSystems(for: item), !systems.isEmpty else { //this is actually an import error @@ -592,7 +608,9 @@ public final class GameImporter: GameImporting, ObservableObject { try await gameImporterFileService.moveImportItem(toAppropriateSubfolder: item) //import the copied file into our database + try await gameImporterDatabaseService.importGameIntoDatabase(queueItem: item) + //if everything went well and no exceptions, we're clear to indicate a successful import // do { // //try moving it to the correct location - we may clean this up later. @@ -617,7 +635,7 @@ public final class GameImporter: GameImporting, ObservableObject { // } // for each //external callers - might not be needed in the end - self.completionHandler?(self.encounteredConflicts) +// self.completionHandler?(self.encounteredConflicts) } // General status update for GameImporter @@ -630,7 +648,6 @@ public final class GameImporter: GameImporting, ObservableObject { /// Checks the queue and all child elements in the queue to see if this file exists. if it does, return true, else return false. private func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool { let duplicate = importQueue.contains { existing in - var exists = false if (existing.url == queueItem.url || existing.id == queueItem.id) { return true } else if (!existing.childQueueItems.isEmpty) { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 3d9fc66473..c3202bd200 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -53,7 +53,13 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { throw GameImporterError.systemNotDetermined } - DLOG("Attempting to import game: \(queueItem.destinationUrl?.lastPathComponent) for system: \(targetSystem.name)") + //TODO: is it an error if we don't have the destination url at this point? + guard let destUrl = queueItem.destinationUrl else { + //how did we get here, throw? + throw GameImporterError.incorrectDestinationURL + } + + DLOG("Attempting to import game: \(destUrl.lastPathComponent) for system: \(targetSystem.name)") let filename = queueItem.url.lastPathComponent let partialPath = (targetSystem.identifier as NSString).appendingPathComponent(filename) @@ -74,6 +80,12 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { /// Imports a ROM to the database internal func importToDatabaseROM(forItem queueItem: ImportQueueItem, system: PVSystem, relatedFiles: [URL]?) async throws { + + guard let _ = queueItem.destinationUrl else { + //how did we get here, throw? + throw GameImporterError.incorrectDestinationURL + } + DLOG("Starting database ROM import for: \(queueItem.url.lastPathComponent)") let filename = queueItem.url.lastPathComponent let filenameSansExtension = queueItem.url.deletingPathExtension().lastPathComponent @@ -82,10 +94,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { let partialPath: String = (system.identifier as NSString).appendingPathComponent(filename) DLOG("Creating game object with title: \(title), partialPath: \(partialPath)") - guard let destUrl = queueItem.destinationUrl else { - //how did we get here, throw? - return - } + let file = PVFile(withURL: queueItem.destinationUrl!) let game = PVGame(withFile: file, system: system) game.romPath = partialPath @@ -296,7 +305,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { //step 1 - calculate md5 hash if needed if game.md5Hash.isEmpty { - if let romFullPath = romsPath?.appendingPathComponent(game.romPath).path { + if let _ = romsPath?.appendingPathComponent(game.romPath).path { if let md5Hash = calculateMD5(forGame: game) { game.md5Hash = md5Hash } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index 0075f956ad..5c8bc775ad 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -1,5 +1,5 @@ // -// File.swift +// GameImporterFileService.swift // PVLibrary // // Created by David Proskin on 11/5/24. diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index 6dddb235ba..78a2393cea 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -1,5 +1,5 @@ // -// File.swift +// GameImporterSystemsService.swift // PVLibrary // // Created by David Proskin on 11/7/24. @@ -25,6 +25,7 @@ protocol GameImporterSystemsServicing { func setOpenVGDB(_ vgdb: OpenVGDB) func setExtensionsToSystemMapping(_ mapping: [String: [String]]) func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] + func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? } class GameImporterSystemsService : GameImporterSystemsServicing { @@ -494,7 +495,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { } /// Finds any current game that could belong to any of the given systems - internal class func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? { + func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? { // Check if existing ROM let allGames = RomDatabase.gamesCache.values.filter ({ From 470bf55888d9c6c21c5e66dcb0f630a66648695e Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 13:37:33 -0500 Subject: [PATCH 08/30] [For Now] - expose the concept of getting artwork via the game importer. ideally that's a utility that need not live here --- .../Importer/Services/GameImporter/GameImporter.swift | 4 ++++ .../Services/GameImporter/GameImporterDatabaseService.swift | 1 + 2 files changed, 5 insertions(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 8f1916ed51..3ab9c3349e 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -317,6 +317,10 @@ public final class GameImporter: GameImporting, ObservableObject { await PVEmulatorConfiguration.updateCores(fromPlists: corePlists) } + public func getArtwork(forGame game: PVGame) async -> PVGame { + return await gameImporterDatabaseService.getArtwork(forGame: game) + } + /// Deinitializer deinit { notificationToken?.invalidate() diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index c3202bd200..bae28409bd 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -25,6 +25,7 @@ protocol GameImporterDatabaseServicing { func setOpenVGDB(_ vgdb: OpenVGDB) func setRomsPath(url:URL) func importGameIntoDatabase(queueItem: ImportQueueItem) async throws + func getArtwork(forGame game: PVGame) async -> PVGame } class GameImporterDatabaseService : GameImporterDatabaseServicing { From 9805f4ef5bdeb269929f138cd871da44cb17398c Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 13:39:21 -0500 Subject: [PATCH 09/30] commenting out some code for now that we need to. update per the new GameImporter interfaces --- .../PVGameLibraryViewController.swift | 3 ++- .../PVGameLibraryUpdatesController.swift | 25 +++++++++++-------- .../PVGameMoreInfoViewController.swift | 3 ++- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift b/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift index 566b3f94fd..ccf4b3d364 100644 --- a/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift @@ -1010,7 +1010,8 @@ public final class PVGameLibraryViewController: GCEventViewController, UITextFie self.hud.hide(animated:true, afterDelay: 1.0) Task.detached(priority: .utility) { do { - try await self.gameImporter.importFiles(atPaths: paths) + //TODO: fix this + //try await self.gameImporter.importFiles(atPaths: paths) } catch { ELOG("Error: \(error.localizedDescription)") } diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift index c05d52e731..688a1aa910 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift @@ -288,7 +288,8 @@ public final class PVGameLibraryUpdatesController: ObservableObject { } if !newGames.isEmpty { ILOG("PVGameLibraryUpdatesController: Importing \(newGames)") - await gameImporter.getRomInfoForFiles(atPaths: newGames, userChosenSystem: system.asDomain()) + //TODO: I think we want to add items to the import queue here + //await gameImporter.getRomInfoForFiles(atPaths: newGames, userChosenSystem: system.asDomain()) #if os(iOS) || os(macOS) || targetEnvironment(macCatalyst) await MainActor.run { Task { @@ -439,7 +440,8 @@ extension PVGameLibraryUpdatesController { extension PVGameLibraryUpdatesController: ConflictsController { public func resolveConflicts(withSolutions solutions: [URL : System]) async { - await gameImporter.resolveConflicts(withSolutions: solutions) + //TODO: fix this +// await gameImporter.resolveConflicts(withSolutions: solutions) await updateConflicts() } @@ -461,15 +463,16 @@ extension PVGameLibraryUpdatesController: ConflictsController { // } let filesInConflictsFolder = conflictsWatcher.conflictFiles - let sortedFiles = PVEmulatorConfiguration.sortImportURLs(urls: filesInConflictsFolder) - - self.conflicts = sortedFiles.compactMap { file -> (path: URL, candidates: [System])? in - let candidates = RomDatabase.systemCache.values - .filter { $0.supportedExtensions.contains(file.pathExtension.lowercased()) } - .map { $0.asDomain() } - - return candidates.isEmpty ? nil : .init((path: file, candidates: candidates)) - } + //TODO: fix alongside conflicts +// let sortedFiles = PVEmulatorConfiguration.sortImportURLs(urls: filesInConflictsFolder) +// +// self.conflicts = sortedFiles.compactMap { file -> (path: URL, candidates: [System])? in +// let candidates = RomDatabase.systemCache.values +// .filter { $0.supportedExtensions.contains(file.pathExtension.lowercased()) } +// .map { $0.asDomain() } +// +// return candidates.isEmpty ? nil : .init((path: file, candidates: candidates)) +// } } } } diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift index 3bda86adf0..6a9996cacb 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift @@ -706,7 +706,8 @@ public final class PVGameMoreInfoViewController: PVGameMoreInfoViewControllerBas if reloadGameInfoAfter, game.releaseID == nil || game.releaseID!.isEmpty { Task { [weak self] in guard let self = self else { return } - self.game = GameImporter.shared.lookupInfo(for: game, overwrite: false) + //TODO: fix this + //self.game = GameImporter.shared.lookupInfo(for: game, overwrite: false) } } } From f154a777066435bc5cac69fd33f3a3f901b31311 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 17:30:52 -0500 Subject: [PATCH 10/30] don't forget to set valid systems when determined on the import item --- .../Importer/Services/GameImporter/GameImporter.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 3ab9c3349e..adf6d0098e 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -600,6 +600,9 @@ public final class GameImporter: GameImporting, ObservableObject { throw GameImporterError.noSystemMatched } + //update item's candidate systems with the result of determineSystems + item.systems = systems + //this might be a conflict if we can't infer what to do if item.systems.count > 1 { //conflict From 140599604306cf7ed9a4e7e89c131a1b004e47a5 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 17:31:34 -0500 Subject: [PATCH 11/30] make sure we move rom to correct directory and not an extra directory --- .../Services/GameImporter/GameImporterFileService.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index 5c8bc775ad..dc588d2745 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -80,10 +80,9 @@ class GameImporterFileService : GameImporterFileServicing { let fileName = queueItem.url.lastPathComponent let destinationFolder = targetSystem.romsDirectory - let destinationPath = destinationFolder.appendingPathComponent(fileName) do { - queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationPath) + queueItem.destinationUrl = try await moveFile(queueItem.url, to: destinationFolder) try await moveChildImports(forQueueItem: queueItem, to: destinationFolder) } catch { throw GameImporterError.failedToMoveROM(error) From 52c0f6a98b6f9d0b62fb0d61b60a1cecc7d67bf1 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 17:32:17 -0500 Subject: [PATCH 12/30] skip hidden files on importer --- .../PVUIBase/Game Library/PVGameLibraryUpdatesController.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift index 688a1aa910..88bcc73527 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift @@ -266,7 +266,8 @@ public final class PVGameLibraryUpdatesController: ObservableObject { do { return try await FileManager.default.contentsOfDirectory(at: path, includingPropertiesForKeys: nil, - options: [.skipsPackageDescendants, .skipsSubdirectoryDescendants]) + options: [.skipsPackageDescendants, .skipsSubdirectoryDescendants, + .skipsHiddenFiles]) } catch { ELOG("Error scanning initial files: \(error)") return [] From 3909ec9ef08df726625fba087368054d8a058f60 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Thu, 7 Nov 2024 17:32:39 -0500 Subject: [PATCH 13/30] Fix toolbar items for game import status view --- .../PVSwiftUI/Imports/ImportStatusView.swift | 24 +++++++++++++------ .../PVSwiftUI/RootView/PVMenuDelegate.swift | 4 ++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index 34a63be506..f4a9454e6b 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -12,6 +12,7 @@ import PVLibrary public protocol ImportStatusDelegate : AnyObject { func dismissAction() func addImportsAction() + func forceImportsAction() } // View Model to manage import tasks @@ -82,13 +83,22 @@ struct ImportStatusView: View { .padding() } .navigationTitle("Import Status") - .navigationBarItems( - leading: Button("Done") { delegate.dismissAction() }, - trailing: - Button("Import Files") { - delegate?.addImportsAction() - } - ) + .toolbar { + ToolbarItemGroup(placement: .topBarLeading, + content: { + Button("Done") { delegate.dismissAction() + } + }) + ToolbarItemGroup(placement: .topBarTrailing, + content: { + Button("Import Files") { + delegate?.addImportsAction() + } + Button("Force Import") { + delegate?.forceImportsAction() + } + }) + } } .presentationDetents([.medium, .large]) .presentationDragIndicator(.visible) diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift index 8f1bf68ace..eca7208d96 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift @@ -241,5 +241,9 @@ extension PVRootViewController: ImportStatusDelegate { public func addImportsAction() { self.showImportOptionsAlert() } + + public func forceImportsAction() { + GameImporter.shared.startProcessing() + } } #endif From 8cfa5246dee0e648d603928be626e93965f11884 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Fri, 8 Nov 2024 12:34:23 -0500 Subject: [PATCH 14/30] fixed some systems detection, simplified to allow a list when we can't be sure of the right system. conflicts are now properly detected and moved to the conflicts folder, and shown in UI if a conflict is detected, user can tap to enter SystemSelectionView which offers up valid systems tapping an item will update that ImportItem's targetSystem so hopefully import works after that. --- .../Importer/Models/GameImporterError.swift | 1 + .../Services/GameImporter/GameImporter.swift | 17 +- .../GameImporterDatabaseService.swift | 3 + .../GameImporterFileService.swift | 1 - .../GameImporterSystemsService.swift | 538 +++++++++--------- .../PVSwiftUI/Imports/ImportStatusView.swift | 48 +- .../Imports/SystemSelectionView.swift | 36 ++ .../PVSwiftUI/RootView/PVMenuDelegate.swift | 7 + 8 files changed, 381 insertions(+), 270 deletions(-) create mode 100644 PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift index 84e9a829cf..7f060adbed 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/GameImporterError.swift @@ -17,4 +17,5 @@ public enum GameImporterError: Error, Sendable { case noBIOSMatchForBIOSFileType case unsupportedCDROMFile case incorrectDestinationURL + case conflictDetected } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index adf6d0098e..5b4cd3baa2 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -551,17 +551,22 @@ public final class GameImporter: GameImporting, ObservableObject { item.status = .success updateImporterStatus("Completed \(item.url.lastPathComponent)") ILOG("GameImportQueue - processing item in queue: \(item.url) completed.") - } catch { - ILOG("GameImportQueue - processing item in queue: \(item.url) caught error...") - if error.localizedDescription.contains("Conflict") { + } catch let error as GameImporterError { + switch error { + case .conflictDetected: item.status = .conflict updateImporterStatus("Conflict for \(item.url.lastPathComponent). User action needed.") WLOG("GameImportQueue - processing item in queue: \(item.url) restuled in conflict.") - } else { + default: item.status = .failure updateImporterStatus("Failed \(item.url.lastPathComponent) with error: \(error.localizedDescription)") ELOG("GameImportQueue - processing item in queue: \(item.url) restuled in error: \(error.localizedDescription)") } + } catch { + ILOG("GameImportQueue - processing item in queue: \(item.url) caught error... \(error.localizedDescription)") + item.status = .failure + updateImporterStatus("Failed \(item.url.lastPathComponent) with error: \(error.localizedDescription)") + ELOG("GameImportQueue - processing item in queue: \(item.url) restuled in error: \(error.localizedDescription)") } } @@ -604,11 +609,13 @@ public final class GameImporter: GameImporting, ObservableObject { item.systems = systems //this might be a conflict if we can't infer what to do - if item.systems.count > 1 { + //for BIOS, we can handle multiple systems, so allow that to proceed + if item.fileType != .bios && item.targetSystem() == nil { //conflict item.status = .conflict //start figuring out what to do, because this item is a conflict try await gameImporterFileService.moveToConflictsFolder(item, conflictsPath: conflictPath) + throw GameImporterError.conflictDetected } //move ImportQueueItem to appropriate file location diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index bae28409bd..b47e807528 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -50,6 +50,9 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } internal func importGameIntoDatabase(queueItem: ImportQueueItem) async throws { + //TODO: what do if this is a BIOS? + + guard let targetSystem = queueItem.targetSystem() else { throw GameImporterError.systemNotDetermined } diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index dc588d2745..7c7086d1b4 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -78,7 +78,6 @@ class GameImporterFileService : GameImporterFileServicing { throw GameImporterError.systemNotDetermined } - let fileName = queueItem.url.lastPathComponent let destinationFolder = targetSystem.romsDirectory do { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index 78a2393cea..72ce5253c1 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -45,67 +45,258 @@ class GameImporterSystemsService : GameImporterSystemsServicing { romExtensionToSystemsMap = mapping } - internal func matchSystemByPartialName(_ fileName: String, possibleSystems: [PVSystem]) -> PVSystem? { - let cleanedName = fileName.lowercased() + /// Determines the system for a given candidate file + public func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] { + guard let md5 = queueItem.md5?.uppercased() else { + throw GameImporterError.couldNotCalculateMD5 + } - for system in possibleSystems { - let patterns = filenamePatterns(forSystem: system) - - for pattern in patterns { - if (try? NSRegularExpression(pattern: pattern, options: .caseInsensitive))? - .firstMatch(in: cleanedName, options: [], range: NSRange(cleanedName.startIndex..., in: cleanedName)) != nil { - DLOG("Found system match by pattern '\(pattern)' for system: \(system.name)") - return system + let fileExtension = queueItem.url.pathExtension.lowercased() + + DLOG("Checking MD5: \(md5) for possible BIOS match") + + // First check if this is a BIOS file by MD5 + if queueItem.fileType == .bios { + let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) + var biosSystemMatches:[PVSystem] = [] + for bios in biosMatches { + biosSystemMatches.append(bios.system) + } + return biosSystemMatches + } + + //not bios or artwork, start narrowing it down. + + if PVEmulatorConfiguration.supportedCDFileExtensions.contains(fileExtension) { + if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + if systems.count == 1 { + return [systems[0]] + } else if systems.count > 1 { + return try determineSystemsFromContent(for: queueItem, possibleSystems: systems) } } } - return nil + // Try to find system by MD5 using OpenVGDB + if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), + let _ = results.first{ + + // Get all matching systems + let matchingSystems = results.compactMap { result -> PVSystem? in + guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } + return PVEmulatorConfiguration.system(forIdentifier: sysID) + } + + //temporarily removing this logic - if we have multiple valid systems, we'll reconcile later. + +// if !matchingSystems.isEmpty { +// // Sort by release year and take the oldest +// //TODO: consider whether this is a good idea? +// if let oldestSystem = matchingSystems.sorted(by: { $0.releaseYear < $1.releaseYear }).first { +// //TODO: is this the right move, i'm not sure - might be better to consider a conflict here +// DLOG("System determined by MD5 match (oldest): \(oldestSystem.name) (\(oldestSystem.releaseYear))") +// return [oldestSystem] +// } +// } + +// // Fallback to original single system match if sorting fails +// if let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { +// DLOG("System determined by MD5 match (fallback): \(system.name)") +// return [system] +// } + return matchingSystems + } + + DLOG("MD5 lookup failed, trying filename matching") + + // Try filename matching next + let fileName = queueItem.url.lastPathComponent + + + let matchedSystems = await matchSystemByFileName(fileName) + if !matchedSystems.isEmpty { + return matchedSystems + } + + // If MD5 lookup fails, try to determine the system based on file extension + if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + if systems.count == 1 { + return systems + } else if systems.count > 1 { + return try determineSystemsFromContent(for: queueItem, possibleSystems: systems) + } + } + + throw GameImporterError.noSystemMatched } - /// Matches a system based on the file name - internal func matchSystemByFileName(_ fileName: String) async -> PVSystem? { - let systems = PVEmulatorConfiguration.systems - let lowercasedFileName = fileName.lowercased() - let fileExtension = (fileName as NSString).pathExtension.lowercased() + /// Determines the system for a given candidate file + private func determineSystemsFromContent(for queueItem: ImportQueueItem, possibleSystems: [PVSystem]) throws -> [PVSystem] { + // Implement logic to determine system based on file content or metadata + // This could involve checking file headers, parsing content, or using a database of known games - // First, try to match based on file extension - if let systemsForExtension = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { - if systemsForExtension.count == 1 { - return systemsForExtension[0] - } else if systemsForExtension.count > 1 { - // If multiple systems match the extension, try to narrow it down - for system in systemsForExtension { - if await doesFileNameMatch(lowercasedFileName, forSystem: system) { - return system + let fileName = queueItem.url.deletingPathExtension().lastPathComponent + + var matchedSystems:[PVSystem] = [] + for system in possibleSystems { + do { + if let results = try openVGDB?.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), + !results.isEmpty { + ILOG("System determined by filename match in OpenVGDB: \(system.name)") + matchedSystems.append(system) + } + } catch { + ELOG("Error searching OpenVGDB for system \(system.name): \(error.localizedDescription)") + } + } + + if (!matchedSystems.isEmpty) { + return matchedSystems + } + + // If we couldn't determine the system, try a more detailed search + if let fileMD5 = queueItem.md5?.uppercased(), !fileMD5.isEmpty { + do { + if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: fileMD5) { + for result in results { + if let systemID = result["systemID"] as? Int, + let system = possibleSystems.first(where: { $0.openvgDatabaseID == systemID }) { + matchedSystems.append(system) + } } + ILOG("Number of Systems matched by MD5 match in OpenVGDB: \(matchedSystems.count)") + return matchedSystems } + } catch { + //what to do here, since this results in no system? + ELOG("Error searching OpenVGDB by MD5: \(error.localizedDescription)") } } - // If extension matching fails, try other methods - for system in systems { - if await doesFileNameMatch(lowercasedFileName, forSystem: system) { - return system + // If still no match, try to determine based on file content + // This is a placeholder for more advanced content-based detection + // You might want to implement system-specific logic here + for system in possibleSystems { + if doesFileContentMatch(queueItem, forSystem: system) { + ILOG("System determined by file content match: \(system.name)") + matchedSystems.append(system) } } - // If no match found, try querying the OpenVGDB - //TODO: fix me + // If we still couldn't determine the system, return the first possible system as a fallback + WLOG("Could not determine system from content, return anything we matched so far - if nothing, return all possible systems") + return matchedSystems.isEmpty ? possibleSystems : matchedSystems + } + + //TODO: is this called? + /// Retrieves the system ID from the cache for a given ROM candidate + internal func systemIdFromCache(forQueueItem queueItem: ImportQueueItem) -> String? { + guard let md5 = queueItem.md5 else { + ELOG("MD5 was blank") + return nil + } + let result = RomDatabase.artMD5DBCache[md5] ?? RomDatabase.getArtCacheByFileName(queueItem.url.lastPathComponent) + + if let _res = result, + let databaseID = _res["systemID"] as? Int, + let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { + return systemID + } + return nil + } + + //TODO: is this called? + /// Matches a system based on the ROM candidate + internal func systemId(forQueueItem queueItem: ImportQueueItem) -> String? { + guard let md5 = queueItem.md5 else { + ELOG("MD5 was blank") + return nil + } + + let fileName: String = queueItem.url.lastPathComponent + do { - if let results = try openVGDB?.searchDatabase(usingFilename: fileName), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? Int, - let system = PVEmulatorConfiguration.system(forDatabaseID: systemID) { - return system + if let databaseID = try openVGDB?.system(forRomMD5: md5, or: fileName), + let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { + return systemID + } else { + ILOG("Could't match \(queueItem.url.lastPathComponent) based off of MD5 {\(md5)}") + return nil } } catch { - ELOG("Error querying OpenVGDB for filename: \(error.localizedDescription)") + DLOG("Unable to find rom by MD5: \(error.localizedDescription)") + return nil + } + } + + //TODO: is this called? + internal func determineSystemByMD5(_ queueItem: ImportQueueItem) async throws -> PVSystem? { + guard let md5 = queueItem.md5?.uppercased() else { + throw GameImporterError.couldNotCalculateMD5 } + DLOG("Attempting MD5 lookup for: \(md5)") + + // Try to find system by MD5 using OpenVGDB + if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), + let firstResult = results.first, + let systemID = firstResult["systemID"] as? NSNumber, + let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { + DLOG("System determined by MD5 match: \(system.name)") + return system + } + + DLOG("No system found by MD5") return nil } + //TODO: is this called? + /// Determines the systems for a given path + internal func determineSystems(for path: URL, chosenSystem: System?) throws -> [PVSystem] { + if let chosenSystem = chosenSystem { + if let system = RomDatabase.systemCache[chosenSystem.identifier] { + return [system] + } + } + + let fileExtensionLower = path.pathExtension.lowercased() + return PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtensionLower) ?? [] + } + + /// Finds any current game that could belong to any of the given systems + func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? { + // Check if existing ROM + + let allGames = RomDatabase.gamesCache.values.filter ({ + $0.romPath.lowercased() == romFilename.lowercased() + }) + /* + let database = RomDatabase.sharedInstance + + let predicate = NSPredicate(format: "romPath CONTAINS[c] %@", PVEmulatorConfiguration.stripDiscNames(fromFilename: romFilename)) + let allGames = database.all(PVGame.self, filter: predicate) + */ + // Optionally filter to specfici systems + if let systems = systems { + //let filteredGames = allGames.filter { systems.contains($0.system) } + var sysIds:[String:Bool]=[:] + systems.forEach({ sysIds[$0.identifier] = true }) + let filteredGames = allGames.filter { sysIds[$0.systemIdentifier] != nil } + return filteredGames.isEmpty ? nil : Array(filteredGames) + } else { + return allGames.isEmpty ? nil : Array(allGames) + } + } + + /// Returns the system identifiers for a given ROM path + public func systemIDsForRom(at path: URL) -> [String]? { + let fileExtension: String = path.pathExtension.lowercased() + return romExtensionToSystemsMap[fileExtension] + } + + + //MARK: Utilities + /// Checks if a file name matches a given system private func doesFileNameMatch(_ lowercasedFileName: String, forSystem system: PVSystem) async -> Bool { // Check if the filename contains the system's name or abbreviation @@ -277,56 +468,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { return patterns } - /// Determines the system for a given candidate file - private func determineSystemFromContent(for queueItem: ImportQueueItem, possibleSystems: [PVSystem]) throws -> PVSystem { - // Implement logic to determine system based on file content or metadata - // This could involve checking file headers, parsing content, or using a database of known games - - let fileName = queueItem.url.deletingPathExtension().lastPathComponent - - for system in possibleSystems { - do { - if let results = try openVGDB?.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), - !results.isEmpty { - ILOG("System determined by filename match in OpenVGDB: \(system.name)") - return system - } - } catch { - ELOG("Error searching OpenVGDB for system \(system.name): \(error.localizedDescription)") - } - } - - // If we couldn't determine the system, try a more detailed search - if let fileMD5 = queueItem.md5?.uppercased(), !fileMD5.isEmpty { - do { - if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: fileMD5), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? Int, - let system = possibleSystems.first(where: { $0.openvgDatabaseID == systemID }) { - ILOG("System determined by MD5 match in OpenVGDB: \(system.name)") - return system - } - } catch { - //what to do here, since this results in no system? - ELOG("Error searching OpenVGDB by MD5: \(error.localizedDescription)") - } - } - - // If still no match, try to determine based on file content - // This is a placeholder for more advanced content-based detection - // You might want to implement system-specific logic here - for system in possibleSystems { - if doesFileContentMatch(queueItem, forSystem: system) { - ILOG("System determined by file content match: \(system.name)") - return system - } - } - - // If we still couldn't determine the system, return the first possible system as a fallback - //TODO: should we actually do this? - WLOG("Could not determine system from content, using first possible system as fallback") - return possibleSystems[0] - } + /// Checks if a file content matches a given system private func doesFileContentMatch(_ queueItem: ImportQueueItem, forSystem system: PVSystem) -> Bool { @@ -336,193 +478,75 @@ class GameImporterSystemsService : GameImporterSystemsServicing { return false } - /// Determines the system for a given candidate file - public func determineSystems(for queueItem: ImportQueueItem) async throws -> [PVSystem] { - guard let md5 = queueItem.md5?.uppercased() else { - throw GameImporterError.couldNotCalculateMD5 - } - - let fileExtension = queueItem.url.pathExtension.lowercased() - - DLOG("Checking MD5: \(md5) for possible BIOS match") - // First check if this is a BIOS file by MD5 - - if queueItem.fileType == .bios { - let biosMatches = PVEmulatorConfiguration.biosEntries.filter("expectedMD5 == %@", md5).map({ $0 }) - var biosSystemMatches:[PVSystem] = [] - for bios in biosMatches { - biosSystemMatches.append(bios.system) - } - return biosSystemMatches - } - - if PVEmulatorConfiguration.supportedCDFileExtensions.contains(fileExtension) { - if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { - if systems.count == 1 { - return [systems[0]] - } else if systems.count > 1 { - // For CD games with multiple possible systems, use content detection - // but allow for the fact that this might not exclude a system - //TODO: fixme - let aSystem = try determineSystemFromContent(for: queueItem, possibleSystems: systems) - return [aSystem] - } - } - } + //TODO: this isn't used remove? + internal func matchSystemByPartialName(_ fileName: String, possibleSystems: [PVSystem]) -> PVSystem? { + let cleanedName = fileName.lowercased() - // Try to find system by MD5 using OpenVGDB - if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? NSNumber { - - // Get all matching systems - let matchingSystems = results.compactMap { result -> PVSystem? in - guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } - return PVEmulatorConfiguration.system(forIdentifier: sysID) - } + for system in possibleSystems { + let patterns = filenamePatterns(forSystem: system) - if !matchingSystems.isEmpty { - // Sort by release year and take the oldest - //TODO: consider whether this is a good idea? - if let oldestSystem = matchingSystems.sorted(by: { $0.releaseYear < $1.releaseYear }).first { - //TODO: is this the right move, i'm not sure - might be better to consider a conflict here - DLOG("System determined by MD5 match (oldest): \(oldestSystem.name) (\(oldestSystem.releaseYear))") - return [oldestSystem] + for pattern in patterns { + if (try? NSRegularExpression(pattern: pattern, options: .caseInsensitive))? + .firstMatch(in: cleanedName, options: [], range: NSRange(cleanedName.startIndex..., in: cleanedName)) != nil { + DLOG("Found system match by pattern '\(pattern)' for system: \(system.name)") + return system } } - - // Fallback to original single system match if sorting fails - if let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { - DLOG("System determined by MD5 match (fallback): \(system.name)") - return [system] - } - } - - DLOG("MD5 lookup failed, trying filename matching") - - // Try filename matching next - let fileName = queueItem.url.lastPathComponent - - if let matchedSystem = await matchSystemByFileName(fileName) { - DLOG("Found system by filename match: \(matchedSystem.name)") - return [matchedSystem] - } - - // If MD5 lookup fails, try to determine the system based on file extension - if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { - if systems.count == 1 { - return systems - } else if systems.count > 1 { - // If multiple systems support this extension, try to determine based on file content or metadata - //TODO: fixme - let aSystem = try determineSystemFromContent(for: queueItem, possibleSystems: systems) - return [aSystem] - } - } - - throw GameImporterError.noSystemMatched - } - - /// Retrieves the system ID from the cache for a given ROM candidate - public func systemIdFromCache(forQueueItem queueItem: ImportQueueItem) -> String? { - guard let md5 = queueItem.md5 else { - ELOG("MD5 was blank") - return nil } - let result = RomDatabase.artMD5DBCache[md5] ?? RomDatabase.getArtCacheByFileName(queueItem.url.lastPathComponent) - if let _res = result, - let databaseID = _res["systemID"] as? Int, - let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { - return systemID - } return nil } - /// Matches a system based on the ROM candidate - public func systemId(forQueueItem queueItem: ImportQueueItem) -> String? { - guard let md5 = queueItem.md5 else { - ELOG("MD5 was blank") - return nil - } - - let fileName: String = queueItem.url.lastPathComponent + /// Matches a system based on the file name + /// Can return multiple possible matches + internal func matchSystemByFileName(_ fileName: String) async -> [PVSystem] { +// let systems = PVEmulatorConfiguration.systems + let lowercasedFileName = fileName.lowercased() + let fileExtension = (fileName as NSString).pathExtension.lowercased() + var validSystems:[PVSystem] = [] - do { - if let databaseID = try openVGDB?.system(forRomMD5: md5, or: fileName), - let systemID = PVEmulatorConfiguration.systemID(forDatabaseID: databaseID) { - return systemID - } else { - ILOG("Could't match \(queueItem.url.lastPathComponent) based off of MD5 {\(md5)}") - return nil + // First, try to match based on file extension + if let systemsForExtension = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + if systemsForExtension.count == 1 { + return [systemsForExtension[0]] + } else if systemsForExtension.count > 1 { + // If multiple systems match the extension, try to narrow it down + for system in systemsForExtension { + if await doesFileNameMatch(lowercasedFileName, forSystem: system) { + validSystems.append(system) + } + } + return validSystems } - } catch { - DLOG("Unable to find rom by MD5: \(error.localizedDescription)") - return nil - } - } - - internal func determineSystemByMD5(_ queueItem: ImportQueueItem) async throws -> PVSystem? { - guard let md5 = queueItem.md5?.uppercased() else { - throw GameImporterError.couldNotCalculateMD5 } - DLOG("Attempting MD5 lookup for: \(md5)") +// //TODO: consider if this is useful +// // If extension matching fails, try checking EVERY system +// for system in systems { +// var validSystems:[PVSystem] = [] +// for system in systemsForExtension { +// if await doesFileNameMatch(lowercasedFileName, forSystem: system) { +// validSystems.append(system) +// } +// } +// return validSystems +// } - // Try to find system by MD5 using OpenVGDB - if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), - let firstResult = results.first, - let systemID = firstResult["systemID"] as? NSNumber, - let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { - DLOG("System determined by MD5 match: \(system.name)") - return system - } - - DLOG("No system found by MD5") - return nil - } - - /// Determines the systems for a given path - internal func determineSystems(for path: URL, chosenSystem: System?) throws -> [PVSystem] { - if let chosenSystem = chosenSystem { - if let system = RomDatabase.systemCache[chosenSystem.identifier] { - return [system] + // If no match found, try querying the OpenVGDB + do { + if let results = try openVGDB?.searchDatabase(usingFilename: fileName) { + for result in results { + if let systemID = result["systemID"] as? Int, + let system = PVEmulatorConfiguration.system(forDatabaseID: systemID) { + validSystems.append(system) + } + } } + } catch { + ELOG("Error querying OpenVGDB for filename: \(error.localizedDescription)") } - let fileExtensionLower = path.pathExtension.lowercased() - return PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtensionLower) ?? [] - } - - /// Finds any current game that could belong to any of the given systems - func findAnyCurrentGameThatCouldBelongToAnyOfTheseSystems(_ systems: [PVSystem]?, romFilename: String) -> [PVGame]? { - // Check if existing ROM - - let allGames = RomDatabase.gamesCache.values.filter ({ - $0.romPath.lowercased() == romFilename.lowercased() - }) - /* - let database = RomDatabase.sharedInstance - - let predicate = NSPredicate(format: "romPath CONTAINS[c] %@", PVEmulatorConfiguration.stripDiscNames(fromFilename: romFilename)) - let allGames = database.all(PVGame.self, filter: predicate) - */ - // Optionally filter to specfici systems - if let systems = systems { - //let filteredGames = allGames.filter { systems.contains($0.system) } - var sysIds:[String:Bool]=[:] - systems.forEach({ sysIds[$0.identifier] = true }) - let filteredGames = allGames.filter { sysIds[$0.systemIdentifier] != nil } - return filteredGames.isEmpty ? nil : Array(filteredGames) - } else { - return allGames.isEmpty ? nil : Array(allGames) - } - } - - /// Returns the system identifiers for a given ROM path - public func systemIDsForRom(at path: URL) -> [String]? { - let fileExtension: String = path.pathExtension.lowercased() - return romExtensionToSystemsMap[fileExtension] + return validSystems } } diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index f4a9454e6b..619ec9e1d5 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -36,9 +36,26 @@ func iconNameForFileType(_ type: FileType) -> String { } } +func iconNameForStatus(_ status: ImportStatus) -> String { + switch status { + + case .queued: + return "xmark.circle.fill" + case .processing: + return "progress.indicator" + case .success: + return "checkmark.circle.fill" + case .failure: + return "exclamationmark.triangle.fill" + case .conflict: + return "exclamationmark.triangle.fill" + } +} + // Individual Import Task Row View struct ImportTaskRowView: View { let item: ImportQueueItem + @State private var isNavigatingToSystemSelection = false var body: some View { HStack { @@ -56,7 +73,7 @@ struct ImportTaskRowView: View { if item.status == .processing { ProgressView().progressViewStyle(.circular).frame(width: 40, height: 40, alignment: .center) } else { - Image(systemName: item.status == .success ? "checkmark.circle.fill" : "xmark.circle.fill") + Image(systemName: iconNameForStatus(item.status)) .foregroundColor(item.status.color) } } @@ -64,6 +81,17 @@ struct ImportTaskRowView: View { .background(Color.white) .cornerRadius(10) .shadow(color: .gray.opacity(0.2), radius: 5, x: 0, y: 2) + .onTapGesture { + if item.status != .conflict { + isNavigatingToSystemSelection = true + } + } + .background( + NavigationLink(destination: SystemSelectionView(item: item), isActive: $isNavigatingToSystemSelection) { + EmptyView() + } + .hidden() + ) } } @@ -75,12 +103,18 @@ struct ImportStatusView: View { var body: some View { NavigationView { ScrollView { - LazyVStack(spacing: 10) { - ForEach(viewModel.gameImporter.importQueue) { item in - ImportTaskRowView(item: item).id(item.id) + if viewModel.gameImporter.importQueue.isEmpty { + Text("No items in the import queue") + .foregroundColor(.gray) + .padding() + } else { + LazyVStack(spacing: 10) { + ForEach(viewModel.gameImporter.importQueue) { item in + ImportTaskRowView(item: item).id(item.id) + } } + .padding() } - .padding() } .navigationTitle("Import Status") .toolbar { @@ -91,10 +125,10 @@ struct ImportStatusView: View { }) ToolbarItemGroup(placement: .topBarTrailing, content: { - Button("Import Files") { + Button("Add Files") { delegate?.addImportsAction() } - Button("Force Import") { + Button("Force") { delegate?.forceImportsAction() } }) diff --git a/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift b/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift new file mode 100644 index 0000000000..68e838f76e --- /dev/null +++ b/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift @@ -0,0 +1,36 @@ +// +// SystemSelectionView.swift +// PVUI +// +// Created by David Proskin on 11/7/24. +// + +import SwiftUI + +struct SystemSelectionView: View { + @ObservedObject var item: ImportQueueItem + @Environment(\.presentationMode) var presentationMode + + var body: some View { + List { + ForEach(item.systems, id: \.self) { system in + Button(action: { + // Set the chosen system and update the status +// item.userChosenSystem = system +// item.status = .queued + // Dismiss the view + presentationMode.wrappedValue.dismiss() + }) { + Text(system.name) + .font(.headline) + .padding() + } + } + } + .navigationTitle("Select System") + } +} + +#Preview { + +} diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift index eca7208d96..9c48d5b8aa 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift @@ -243,6 +243,13 @@ extension PVRootViewController: ImportStatusDelegate { } public func forceImportsAction() { + //reset the status of each item that conflict or failed so we can try again. + GameImporter.shared.importQueue.forEach { item in + if (item.status == .failure || item.status == .conflict) { + item.status = .queued + } + } + GameImporter.shared.startProcessing() } } From 658565b56b01c9e12ef061ef43a21b92484b822d Mon Sep 17 00:00:00 2001 From: David Proskin Date: Fri, 8 Nov 2024 12:57:28 -0500 Subject: [PATCH 15/30] conflicts now detected properly and able to select a chosen system to import to. --- .../Sources/PVLibrary/Importer/Models/ImportQueueItem.swift | 4 ++-- .../Services/GameImporter/GameImporterFileService.swift | 3 ++- PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift | 2 +- PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift index e0032ef676..df666d1658 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Models/ImportQueueItem.swift @@ -52,10 +52,10 @@ public enum ProcessingState { @Observable public class ImportQueueItem: Identifiable, ObservableObject { public let id = UUID() - public let url: URL + public var url: URL public var fileType: FileType public var systems: [PVSystem] // Can be set to the specific system type - public var userChosenSystem: System? + public var userChosenSystem: PVSystem? public var destinationUrl: URL? //this is used when a single import has child items - e.g., m3u, cue, directory diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index 7c7086d1b4..d58adf0917 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -113,7 +113,8 @@ class GameImporterFileService : GameImporterFileServicing { /// Moves a file to the conflicts directory internal func moveToConflictsFolder(_ queueItem: ImportQueueItem, conflictsPath: URL) async throws { let destination = conflictsPath.appendingPathComponent(queueItem.url.lastPathComponent) - queueItem.destinationUrl = try moveAndOverWrite(sourcePath: queueItem.url, destinationPath: destination) + //when moving the conflicts folder, we actually want to update the import item's source url to match + queueItem.url = try moveAndOverWrite(sourcePath: queueItem.url, destinationPath: destination) } /// Move a `URL` to a destination, creating the destination directory if needed diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index 619ec9e1d5..7109cef3ca 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -82,7 +82,7 @@ struct ImportTaskRowView: View { .cornerRadius(10) .shadow(color: .gray.opacity(0.2), radius: 5, x: 0, y: 2) .onTapGesture { - if item.status != .conflict { + if item.status == .conflict { isNavigatingToSystemSelection = true } } diff --git a/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift b/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift index 68e838f76e..4593d81a5b 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/SystemSelectionView.swift @@ -16,8 +16,8 @@ struct SystemSelectionView: View { ForEach(item.systems, id: \.self) { system in Button(action: { // Set the chosen system and update the status -// item.userChosenSystem = system -// item.status = .queued + item.userChosenSystem = system + item.status = .queued // Dismiss the view presentationMode.wrappedValue.dismiss() }) { From bc64ace7f191e790008a57b8e3c6eb910b92c1c4 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Fri, 8 Nov 2024 18:11:36 -0500 Subject: [PATCH 16/30] start to add support for BIOS import processing fix case where looking up in openvgb finds a match - we handle this correctly now. --- .../Services/GameImporter/GameImporter.swift | 9 +++++++-- .../GameImporterDatabaseService.swift | 15 +++++++++++++-- .../GameImporter/GameImporterSystemsService.swift | 15 +++++++++------ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 5b4cd3baa2..403e366df1 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -602,6 +602,7 @@ public final class GameImporter: GameImporting, ObservableObject { guard let systems = try? await gameImporterSystemsService.determineSystems(for: item), !systems.isEmpty else { //this is actually an import error item.status = .failure + ELOG("No system matched for this Import Item: \(item.url.lastPathComponent)") throw GameImporterError.noSystemMatched } @@ -621,8 +622,12 @@ public final class GameImporter: GameImporting, ObservableObject { //move ImportQueueItem to appropriate file location try await gameImporterFileService.moveImportItem(toAppropriateSubfolder: item) - //import the copied file into our database - try await gameImporterDatabaseService.importGameIntoDatabase(queueItem: item) + if (item.fileType == .bios) { + try await gameImporterDatabaseService.importBIOSIntoDatabase(queueItem: item) + } else { + //import the copied file into our database + try await gameImporterDatabaseService.importGameIntoDatabase(queueItem: item) + } //if everything went well and no exceptions, we're clear to indicate a successful import diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index b47e807528..1e781670f8 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -25,6 +25,7 @@ protocol GameImporterDatabaseServicing { func setOpenVGDB(_ vgdb: OpenVGDB) func setRomsPath(url:URL) func importGameIntoDatabase(queueItem: ImportQueueItem) async throws + func importBIOSIntoDatabase(queueItem: ImportQueueItem) async throws func getArtwork(forGame game: PVGame) async -> PVGame } @@ -50,8 +51,9 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } internal func importGameIntoDatabase(queueItem: ImportQueueItem) async throws { - //TODO: what do if this is a BIOS? - + guard queueItem.fileType != .bios else { + return + } guard let targetSystem = queueItem.targetSystem() else { throw GameImporterError.systemNotDetermined @@ -82,6 +84,15 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } } + internal func importBIOSIntoDatabase(queueItem: ImportQueueItem) async throws { + guard let _ = queueItem.destinationUrl else { + //how did we get here, throw? + throw GameImporterError.incorrectDestinationURL + } + + + } + /// Imports a ROM to the database internal func importToDatabaseROM(forItem queueItem: ImportQueueItem, system: PVSystem, relatedFiles: [URL]?) async throws { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index 72ce5253c1..893b54e328 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -62,13 +62,16 @@ class GameImporterSystemsService : GameImporterSystemsServicing { for bios in biosMatches { biosSystemMatches.append(bios.system) } + DLOG("BIOS Match found, returning \(biosSystemMatches.count) valid systems") return biosSystemMatches } //not bios or artwork, start narrowing it down. if PVEmulatorConfiguration.supportedCDFileExtensions.contains(fileExtension) { + DLOG("Possible CD ROM - checking valid systems...") if let systems = PVEmulatorConfiguration.systemsFromCache(forFileExtension: fileExtension) { + DLOG("Possible CD ROM - checking valid systems...found \(systems.count) matches") if systems.count == 1 { return [systems[0]] } else if systems.count > 1 { @@ -83,8 +86,8 @@ class GameImporterSystemsService : GameImporterSystemsServicing { // Get all matching systems let matchingSystems = results.compactMap { result -> PVSystem? in - guard let sysID = (result["systemID"] as? NSNumber).map(String.init) else { return nil } - return PVEmulatorConfiguration.system(forIdentifier: sysID) + guard let sysID = (result["systemID"] as? NSNumber) else { return nil } + return PVEmulatorConfiguration.system(forDatabaseID: sysID.intValue) } //temporarily removing this logic - if we have multiple valid systems, we'll reconcile later. @@ -112,7 +115,6 @@ class GameImporterSystemsService : GameImporterSystemsServicing { // Try filename matching next let fileName = queueItem.url.lastPathComponent - let matchedSystems = await matchSystemByFileName(fileName) if !matchedSystems.isEmpty { return matchedSystems @@ -127,6 +129,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { } } + ELOG("No System matched for this rom: \(fileName)") throw GameImporterError.noSystemMatched } @@ -142,7 +145,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { do { if let results = try openVGDB?.searchDatabase(usingFilename: fileName, systemID: system.openvgDatabaseID), !results.isEmpty { - ILOG("System determined by filename match in OpenVGDB: \(system.name)") + DLOG("System determined by filename match in OpenVGDB: \(system.name)") matchedSystems.append(system) } } catch { @@ -164,7 +167,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { matchedSystems.append(system) } } - ILOG("Number of Systems matched by MD5 match in OpenVGDB: \(matchedSystems.count)") + DLOG("Number of Systems matched by MD5 match in OpenVGDB: \(matchedSystems.count)") return matchedSystems } } catch { @@ -178,7 +181,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { // You might want to implement system-specific logic here for system in possibleSystems { if doesFileContentMatch(queueItem, forSystem: system) { - ILOG("System determined by file content match: \(system.name)") + DLOG("System determined by file content match: \(system.name)") matchedSystems.append(system) } } From b1ec580529a65fe1a0c43a8cfa0808870ec43189 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Fri, 8 Nov 2024 18:13:54 -0500 Subject: [PATCH 17/30] fixes another case where looking up in openvgb would fail --- .../Services/GameImporter/GameImporterSystemsService.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index 893b54e328..37b6591b42 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -244,7 +244,7 @@ class GameImporterSystemsService : GameImporterSystemsServicing { if let results = try openVGDB?.searchDatabase(usingKey: "romHashMD5", value: md5), let firstResult = results.first, let systemID = firstResult["systemID"] as? NSNumber, - let system = PVEmulatorConfiguration.system(forIdentifier: String(systemID.intValue)) { + let system = PVEmulatorConfiguration.system(forDatabaseID: systemID.intValue) { DLOG("System determined by MD5 match: \(system.name)") return system } From 0c3c3999cf74832e315f98755a1961a9e20c6e7f Mon Sep 17 00:00:00 2001 From: David Proskin Date: Fri, 8 Nov 2024 18:36:49 -0500 Subject: [PATCH 18/30] bios file copy works, but crashing on finalizing the import --- .../GameImporterDatabaseService.swift | 27 ++++++++++++++++++- .../GameImporterFileService.swift | 17 +++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 1e781670f8..633717c0ad 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -85,12 +85,37 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } internal func importBIOSIntoDatabase(queueItem: ImportQueueItem) async throws { - guard let _ = queueItem.destinationUrl else { + guard let destinationUrl = queueItem.destinationUrl, + let md5 = queueItem.md5?.uppercased() else { //how did we get here, throw? throw GameImporterError.incorrectDestinationURL } + // Get all BIOS entries that match this MD5 + let matchingBIOSEntries = PVEmulatorConfiguration.biosEntries.filter { biosEntry in + let frozenBiosEntry = biosEntry.isFrozen ? biosEntry : biosEntry.freeze() + return frozenBiosEntry.expectedMD5.uppercased() == md5 + } + + for biosEntry in matchingBIOSEntries { + // Get the first matching system + if let firstBIOSEntry = matchingBIOSEntries.first { + let frozenBiosEntry = firstBIOSEntry.isFrozen ? firstBIOSEntry : firstBIOSEntry.freeze() + + // Update BIOS entry in Realm + try await MainActor.run { + let realm = try Realm() + try realm.write { + if let thawedBios = frozenBiosEntry.thaw() { + let biosFile = PVFile(withURL: destinationUrl) + thawedBios.file = biosFile + } + } + } + } + } + return } /// Imports a ROM to the database diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index d58adf0917..6340930212 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -57,7 +57,7 @@ class GameImporterFileService : GameImporterFileServicing { let biosPath = PVEmulatorConfiguration.biosPath(forSystemIdentifier: system.identifier) .appendingPathComponent(bios.expectedFilename) - queueItem.destinationUrl = try await moveFile(queueItem.url, to: biosPath) + queueItem.destinationUrl = try await moveFile(queueItem.url, toExplicitDestination: biosPath) } } } @@ -118,14 +118,23 @@ class GameImporterFileService : GameImporterFileServicing { } /// Move a `URL` to a destination, creating the destination directory if needed - private func moveFile(_ file: URL, to destination: URL) async throws -> URL { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let destPath = destination.appendingPathComponent(file.lastPathComponent) + private func moveFile(_ file: URL, to destinationDirectory: URL) async throws -> URL { + try FileManager.default.createDirectory(at: destinationDirectory, withIntermediateDirectories: true) + let destPath = destinationDirectory.appendingPathComponent(file.lastPathComponent) try FileManager.default.moveItem(at: file, to: destPath) DLOG("Moved file to: \(destPath.path)") return destPath } + /// Move a `URL` to a destination, creating the destination directory if needed + private func moveFile(_ file: URL, toExplicitDestination destination: URL) async throws -> URL { + let destinationDirectory = destination.deletingLastPathComponent() + try FileManager.default.createDirectory(at: destinationDirectory, withIntermediateDirectories: true) + try FileManager.default.moveItem(at: file, to: destination) + DLOG("Moved file to: \(destination.path)") + return destination + } + /// Moves a file and overwrites if it already exists at the destination public func moveAndOverWrite(sourcePath: URL, destinationPath: URL) throws -> URL { let fileManager = FileManager.default From 23485fae88559d6fd51e026f484946976d4bc2ea Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sat, 9 Nov 2024 06:10:33 -0500 Subject: [PATCH 19/30] removing auto-import start to slow things down adds in a crucial preProcessQueue step to ensure the queue handles cue/bin files correctly remove extraneous view model in ImportStatusView Fixes some bad cases for CDROMs where child items weren't copied to the correct folder Added some more logging in GameImporter --- .../Services/GameImporter/GameImporter.swift | 20 ++++++++++++++----- .../GameImporterFileService.swift | 7 +++++-- .../PVSwiftUI/Imports/ImportStatusView.swift | 11 +++------- .../PVSwiftUI/RootView/PVMenuDelegate.swift | 2 +- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 403e366df1..c666e0d791 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -332,7 +332,7 @@ public final class GameImporter: GameImporting, ObservableObject { public func addImport(_ item: ImportQueueItem) { addImportItemToQueue(item) - startProcessing() +// startProcessing() } public func addImports(forPaths paths: [URL]) { @@ -340,14 +340,16 @@ public final class GameImporter: GameImporting, ObservableObject { addImportItemToQueue(ImportQueueItem(url: url, fileType: .unknown)) }) - startProcessing() +// startProcessing() } // Public method to manually start processing if needed public func startProcessing() { // Only start processing if it's not already active guard processingState == .idle else { return } + self.processingState = .processing Task { + await preProcessQueue() await processQueue() } } @@ -361,6 +363,7 @@ public final class GameImporter: GameImporting, ObservableObject { do { importItem.fileType = try determineImportType(importItem) } catch { + ELOG("Caught error trying to assign file type \(error.localizedDescription)") //caught an error trying to assign file type } @@ -417,22 +420,26 @@ public final class GameImporter: GameImporting, ObservableObject { if let binIndex = importQueue.firstIndex(where: { item in item.url == candidateBinUrl }) { + DLOG("Located corresponding .bin for cue \(baseFileName) - re-parenting queue item") // Remove the .bin item from the queue and add it as a child of the .cue item let binItem = importQueue.remove(at: binIndex) binItem.fileType = .cdRom cueItem.childQueueItems.append(binItem) + } else { + WLOG("Located the corresponding bin[s] for \(baseFileName) - but no corresponding QueueItem detected. Consider creating one here?") } } else { //this is probably some kind of error... - ELOG("Found a .cue \(baseFileName) without a .bin - probably bad things happening") + ELOG("Found a .cue \(baseFileName) without a .bin - probably file system didn't settle yet") } } catch { - ELOG("Caught an error looking for a corresponding .bin to \(baseFileName) - probably bad things happening") + ELOG("Caught an error looking for a corresponding .bin to \(baseFileName) - probably bad things happening - \(error.localizedDescription)") } } } private func findAssociatedBinFile(for cueFileItem: ImportQueueItem) throws -> URL? { + //TODO: handle multi-bin cue let cueContents = try String(contentsOf: cueFileItem.url, encoding: .utf8) let lines = cueContents.components(separatedBy: .newlines) @@ -665,9 +672,12 @@ public final class GameImporter: GameImporting, ObservableObject { } /// Checks the queue and all child elements in the queue to see if this file exists. if it does, return true, else return false. + /// Duplicates are considered if the filename, id, or md5 matches private func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool { let duplicate = importQueue.contains { existing in - if (existing.url == queueItem.url || existing.id == queueItem.id) { + if (existing.url.lastPathComponent.lowercased() == queueItem.url.lastPathComponent.lowercased() + || existing.id == queueItem.id + || existing.md5?.uppercased() == queueItem.md5?.uppercased()) { return true } else if (!existing.childQueueItems.isEmpty) { //check the child queue items for duplicates diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index 6340930212..99d9fcf846 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -97,10 +97,9 @@ class GameImporterFileService : GameImporterFileServicing { for childQueueItem in queueItem.childQueueItems { let fileName = childQueueItem.url.lastPathComponent - let destinationPath = destinationFolder.appendingPathComponent(fileName) do { - childQueueItem.destinationUrl = try await moveFile(childQueueItem.url, to: destinationPath) + childQueueItem.destinationUrl = try await moveFile(childQueueItem.url, to: destinationFolder) //call recursively to keep moving child items to the target directory as a unit try await moveChildImports(forQueueItem: childQueueItem, to: destinationFolder) } catch { @@ -113,8 +112,12 @@ class GameImporterFileService : GameImporterFileServicing { /// Moves a file to the conflicts directory internal func moveToConflictsFolder(_ queueItem: ImportQueueItem, conflictsPath: URL) async throws { let destination = conflictsPath.appendingPathComponent(queueItem.url.lastPathComponent) + DLOG("Moving \(queueItem.url.lastPathComponent) to conflicts folder") //when moving the conflicts folder, we actually want to update the import item's source url to match queueItem.url = try moveAndOverWrite(sourcePath: queueItem.url, destinationPath: destination) + for childQueueItem in queueItem.childQueueItems { + try await moveToConflictsFolder(childQueueItem, conflictsPath: conflictsPath) + } } /// Move a `URL` to a destination, creating the destination directory if needed diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index 7109cef3ca..e8de5d9b5e 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -15,11 +15,6 @@ public protocol ImportStatusDelegate : AnyObject { func forceImportsAction() } -// View Model to manage import tasks -class ImportViewModel: ObservableObject { - public let gameImporter = GameImporter.shared -} - func iconNameForFileType(_ type: FileType) -> String { switch type { @@ -97,19 +92,19 @@ struct ImportTaskRowView: View { struct ImportStatusView: View { @ObservedObject var updatesController: PVGameLibraryUpdatesController - var viewModel:ImportViewModel + var gameImporter:GameImporter weak var delegate:ImportStatusDelegate! var body: some View { NavigationView { ScrollView { - if viewModel.gameImporter.importQueue.isEmpty { + if gameImporter.importQueue.isEmpty { Text("No items in the import queue") .foregroundColor(.gray) .padding() } else { LazyVStack(spacing: 10) { - ForEach(viewModel.gameImporter.importQueue) { item in + ForEach(gameImporter.importQueue) { item in ImportTaskRowView(item: item).id(item.id) } } diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift index 9c48d5b8aa..c1c718a1c9 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift @@ -48,7 +48,7 @@ public protocol PVMenuDelegate: AnyObject { extension PVRootViewController: PVMenuDelegate { public func didTapImports() { - let settingsView = ImportStatusView(updatesController:updatesController, viewModel: ImportViewModel(), delegate: self) + let settingsView = ImportStatusView(updatesController:updatesController, gameImporter: GameImporter.shared, delegate: self) let hostingController = UIHostingController(rootView: settingsView) let navigationController = UINavigationController(rootViewController: hostingController) From c2fac6544c0a6aa38e8746e2fc9c1c21e5e40e40 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sat, 9 Nov 2024 06:36:31 -0500 Subject: [PATCH 20/30] include handling for multi-bin cue files (not tested yet) --- .../Services/GameImporter/GameImporter.swift | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index c666e0d791..3cbc0f01c2 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -415,18 +415,21 @@ public final class GameImporter: GameImporting, ObservableObject { let baseFileName = cueItem.url.deletingPathExtension().lastPathComponent do { - if let candidateBinUrl = try self.findAssociatedBinFile(for: cueItem) { - // Find any .bin item in the queue that matches the .cue base file name - if let binIndex = importQueue.firstIndex(where: { item in - item.url == candidateBinUrl - }) { - DLOG("Located corresponding .bin for cue \(baseFileName) - re-parenting queue item") - // Remove the .bin item from the queue and add it as a child of the .cue item - let binItem = importQueue.remove(at: binIndex) - binItem.fileType = .cdRom - cueItem.childQueueItems.append(binItem) - } else { - WLOG("Located the corresponding bin[s] for \(baseFileName) - but no corresponding QueueItem detected. Consider creating one here?") + let candidateBinUrls = try self.findAssociatedBinFiles(for: cueItem) + if !candidateBinUrls.isEmpty { + for candidateBinUrl in candidateBinUrls { + // Find any .bin item in the queue that matches the .cue base file name + if let binIndex = importQueue.firstIndex(where: { item in + item.url == candidateBinUrl + }) { + DLOG("Located corresponding .bin for cue \(baseFileName) - re-parenting queue item") + // Remove the .bin item from the queue and add it as a child of the .cue item + let binItem = importQueue.remove(at: binIndex) + binItem.fileType = .cdRom + cueItem.childQueueItems.append(binItem) + } else { + WLOG("Located the corresponding bin[s] for \(baseFileName) - but no corresponding QueueItem detected. Consider creating one here?") + } } } else { //this is probably some kind of error... @@ -438,31 +441,37 @@ public final class GameImporter: GameImporting, ObservableObject { } } - private func findAssociatedBinFile(for cueFileItem: ImportQueueItem) throws -> URL? { - //TODO: handle multi-bin cue + private func findAssociatedBinFiles(for cueFileItem: ImportQueueItem) throws -> [URL] { + // Read the contents of the .cue file let cueContents = try String(contentsOf: cueFileItem.url, encoding: .utf8) let lines = cueContents.components(separatedBy: .newlines) - // Look for FILE "something.bin" BINARY line + // Array to hold multiple .bin file URLs + var binFiles: [URL] = [] + + // Look for each line with FILE "something.bin" BINARY for line in lines { let components = line.trimmingCharacters(in: .whitespaces) .components(separatedBy: "\"") + guard components.count >= 2, line.lowercased().contains("file") && line.lowercased().contains("binary") else { continue } + // Extract the .bin file name let binFileName = components[1] let binPath = cueFileItem.url.deletingLastPathComponent().appendingPathComponent(binFileName) + // Check if the .bin file exists and add to the array if it does if FileManager.default.fileExists(atPath: binPath.path) { - return binPath + binFiles.append(binPath) } } - return nil + // Return all found .bin file URLs, or an empty array if none found + return binFiles } - internal func cmpSpecialExt(obj1Extension: String, obj2Extension: String) -> Bool { if obj1Extension == "m3u" && obj2Extension != "m3u" { From 58fc84db03a4975e7b4adfa07cbf1675bd48ae8e Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sat, 9 Nov 2024 10:01:51 -0500 Subject: [PATCH 21/30] fixes some issues with duplicate queue item detection (adds some unit tests too) --- .../Services/GameImporter/GameImporter.swift | 21 ++++++-- .../PVLibraryTests/GameImporterTests.swift | 50 +++++++++++++++++-- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 3cbc0f01c2..efa7378399 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -114,6 +114,8 @@ public protocol GameImporting { func addImport(_ item: ImportQueueItem) func addImports(forPaths paths: [URL]) func startProcessing() + + func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool } @@ -682,13 +684,22 @@ public final class GameImporter: GameImporting, ObservableObject { /// Checks the queue and all child elements in the queue to see if this file exists. if it does, return true, else return false. /// Duplicates are considered if the filename, id, or md5 matches - private func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool { - let duplicate = importQueue.contains { existing in + public func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool { + let duplicate = queue.contains { existing in if (existing.url.lastPathComponent.lowercased() == queueItem.url.lastPathComponent.lowercased() - || existing.id == queueItem.id - || existing.md5?.uppercased() == queueItem.md5?.uppercased()) { + || existing.id == queueItem.id) + { + return true + } + + if let eMd5 = existing.md5?.uppercased(), + let newMd5 = queueItem.md5?.uppercased(), + eMd5 == newMd5 + { return true - } else if (!existing.childQueueItems.isEmpty) { + } + + if (!existing.childQueueItems.isEmpty) { //check the child queue items for duplicates return self.importQueueContainsDuplicate(existing.childQueueItems, ofItem: queueItem) } diff --git a/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift b/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift index ad448369bc..c26c40f627 100644 --- a/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift +++ b/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift @@ -15,7 +15,9 @@ class GameImporterTests: XCTestCase { override func setUp() async throws { try await super.setUp() // Put setup code here. This method is called before the invocation of each test method in the class. - gameImporter = GameImporter(FileManager.default) + //bad, but needed for my test case + //TODO: mock this + gameImporter = GameImporter.shared // await gameImporter.initSystems() <--- this will crash until we get proper DI } @@ -24,9 +26,47 @@ class GameImporterTests: XCTestCase { super.tearDown() gameImporter = nil } + + func testImportQueueContainsDuplicate_noDuplicates() { + let item1 = ImportQueueItem(url: URL(string: "file:///path/to/file1.rom")!) + let item2 = ImportQueueItem(url: URL(string: "file:///path/to/file2.rom")!) + + let queue = [item1] + + XCTAssertFalse(gameImporter.importQueueContainsDuplicate(queue, ofItem: item2), "No duplicates should be found") + } + + func testImportQueueContainsDuplicate_duplicateByUrl() { + let item1 = ImportQueueItem(url: URL(string: "file:///path/to/file1.rom")!) + let item2 = ImportQueueItem(url: URL(string: "file:///path/to/file1.rom")!) + + let queue = [item1] + + XCTAssertTrue(gameImporter.importQueueContainsDuplicate(queue, ofItem: item2), "Duplicate should be detected by URL") + } - func testImportSingleGame_Success() { - // Arrange - let testData = "Test Game Data".data(using: .utf8) - } + func testImportQueueContainsDuplicate_duplicateById() { + let item1 = ImportQueueItem(url: URL(string: "file:///path/to/file1.rom")!) + let item2 = item1 + item2.url = URL(string: "file:///path/to/file2.rom")! + + + let queue = [item1] + + XCTAssertTrue(gameImporter.importQueueContainsDuplicate(queue, ofItem: item2), "Duplicate should be detected by ID") + } + + func testImportQueueContainsDuplicate_duplicateInChildItems() { + let item1 = ImportQueueItem(url: URL(string: "file:///path/to/file1.rom")!) + let item2 = ImportQueueItem(url: URL(string: "file:///path/to/file2.rom")!) + + let child1 = ImportQueueItem(url: URL(string: "file:///path/to/file2.rom")!) + + item1.childQueueItems.append(child1) + + let queue = [item1] + + XCTAssertTrue(gameImporter.importQueueContainsDuplicate(queue, ofItem: item2), "Duplicate should be detected in child queue items") + } } + From 82db386d9e213a65e5a9b7dfbe132efb0f45e9f2 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 08:11:37 -0500 Subject: [PATCH 22/30] adds a lock to prevent re-entry and duplicate adds to the importqueue --- .../Services/GameImporter/GameImporter.swift | 18 ++++++-- .../PVLibraryTests/GameImporterTests.swift | 44 +++++++++++++++++++ .../PVSwiftUI/Imports/ImportStatusView.swift | 28 ++++++++---- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index efa7378399..98f31e6220 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -330,18 +330,27 @@ public final class GameImporter: GameImporting, ObservableObject { //MARK: Public Queue Management + // Inside your GameImporter class + private let importQueueLock = NSLock() + // Adds an ImportItem to the queue without starting processing public func addImport(_ item: ImportQueueItem) { - addImportItemToQueue(item) + importQueueLock.lock() + defer { importQueueLock.unlock() } + + self.addImportItemToQueue(item) + // startProcessing() } public func addImports(forPaths paths: [URL]) { - paths.forEach({ (url) in - addImportItemToQueue(ImportQueueItem(url: url, fileType: .unknown)) - }) + importQueueLock.lock() + defer { importQueueLock.unlock() } + for path in paths { + self.addImportItemToQueue(ImportQueueItem(url: path, fileType: .unknown)) + } // startProcessing() } @@ -703,6 +712,7 @@ public final class GameImporter: GameImporting, ObservableObject { //check the child queue items for duplicates return self.importQueueContainsDuplicate(existing.childQueueItems, ofItem: queueItem) } + DLOG("Duplicate Queue Item not detected for \(existing.url.lastPathComponent.lowercased()) - compared with \(queueItem.url.lastPathComponent.lowercased())") return false } diff --git a/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift b/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift index c26c40f627..9928b0d073 100644 --- a/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift +++ b/PVLibrary/Tests/PVLibraryTests/GameImporterTests.swift @@ -68,5 +68,49 @@ class GameImporterTests: XCTestCase { XCTAssertTrue(gameImporter.importQueueContainsDuplicate(queue, ofItem: item2), "Duplicate should be detected in child queue items") } + + func testImportQueueContainsDuplicate_duplicateByUrlWithSpaces() { + let item1 = ImportQueueItem(url: URL(string: "file:///path/to/Star%20Control%20II%20(USA).bin")!) + let item2 = ImportQueueItem(url: URL(string: "file:///path/to/Star%20Control%20II%20(USA).bin")!) + + let queue = [item1] + + XCTAssertTrue(gameImporter.importQueueContainsDuplicate(queue, ofItem: item2), "Duplicate should be detected by URL") + } + + func testAddImportsThreadSafety() { + // Define paths to test + let paths = [ + URL(string: "file:///path/to/file1.bin")!, + URL(string: "file:///path/to/file2.bin")!, + URL(string: "file:///path/to/file3.bin")! + ] + + // Create an expectation for each concurrent call + let expectation1 = expectation(description: "Thread 1") + let expectation2 = expectation(description: "Thread 2") + let expectation3 = expectation(description: "Thread 3") + + // Dispatch the calls concurrently + DispatchQueue.global(qos: .userInitiated).async { + self.gameImporter.addImports(forPaths: paths) + expectation1.fulfill() + } + + DispatchQueue.global(qos: .userInitiated).async { + self.gameImporter.addImports(forPaths: paths) + expectation2.fulfill() + } + + DispatchQueue.global(qos: .userInitiated).async { + self.gameImporter.addImports(forPaths: paths) + expectation3.fulfill() + } + + // Wait for expectations + wait(for: [expectation1, expectation2, expectation3], timeout: 5.0) + + XCTAssertEqual(gameImporter.importQueue.count, 3, "Expected successful import of all 3 items") + } } diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index e8de5d9b5e..a031da5310 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -58,19 +58,31 @@ struct ImportTaskRowView: View { VStack(alignment: .leading) { Text(item.url.lastPathComponent) .font(.headline) - Text(item.status.description) - .font(.subheadline) - .foregroundColor(item.status.color) + if let targetSystem = item.targetSystem() { + Text(targetSystem.name) + .font(.subheadline) + .foregroundColor(item.status.color) + } + } Spacer() - if item.status == .processing { - ProgressView().progressViewStyle(.circular).frame(width: 40, height: 40, alignment: .center) - } else { - Image(systemName: iconNameForStatus(item.status)) - .foregroundColor(item.status.color) + VStack(alignment: .trailing) { + if item.status == .processing { + ProgressView().progressViewStyle(.circular).frame(width: 40, height: 40, alignment: .center) + } else { + Image(systemName: iconNameForStatus(item.status)) + .foregroundColor(item.status.color) + } + + if (item.childQueueItems.count > 0) { + Text("\(item.childQueueItems.count) assoc. files") + .font(.subheadline) + .foregroundColor(item.status.color) + } } + } .padding() .background(Color.white) From ab29bb1ef621a9c978e4fd4c790100f6a3220a75 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 19:31:22 -0500 Subject: [PATCH 23/30] ImportStatus ui shows number of systems for conflict file, correct system when identified, and "BIOS" for bios (since that was crashing for some reason) --- .../PVSwiftUI/Imports/ImportStatusView.swift | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index a031da5310..b4c8eeb6de 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -58,10 +58,19 @@ struct ImportTaskRowView: View { VStack(alignment: .leading) { Text(item.url.lastPathComponent) .font(.headline) - if let targetSystem = item.targetSystem() { + if item.fileType == .bios { + Text("BIOS") + .font(.subheadline) + .foregroundColor(item.status.color) + } + else if let targetSystem = item.targetSystem() { Text(targetSystem.name) .font(.subheadline) .foregroundColor(item.status.color) + } else if !item.systems.isEmpty { + Text("\(item.systems.count) systems") + .font(.subheadline) + .foregroundColor(item.status.color) } } @@ -77,7 +86,7 @@ struct ImportTaskRowView: View { } if (item.childQueueItems.count > 0) { - Text("\(item.childQueueItems.count) assoc. files") + Text("+\(item.childQueueItems.count) files") .font(.subheadline) .foregroundColor(item.status.color) } From f20bbb3ebcb44600f54d3c52f65e0fd5b4237472 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 19:53:39 -0500 Subject: [PATCH 24/30] [kinda hacky but] dismiss the import status view before showing the import files options --- PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift index c1c718a1c9..4379642fe9 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVMenuDelegate.swift @@ -103,12 +103,14 @@ extension PVRootViewController: PVMenuDelegate { documentPicker = UIDocumentPickerViewController(forOpeningContentTypes: utis, asCopy: true) documentPicker.allowsMultipleSelection = true documentPicker.delegate = self + self.dismiss(animated: true) self.present(documentPicker, animated: true, completion: nil) })) #endif #if canImport(PVWebServer) let webServerAction = UIAlertAction(title: "Web Server", style: .default, handler: { _ in + self.dismiss(animated: true) self.startWebServer() }) From 374ce6a33af9719ff52d4982bb5a284c91f6a84d Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 19:59:14 -0500 Subject: [PATCH 25/30] remove system from view because realm crashes --- PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift | 5 ----- 1 file changed, 5 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index b4c8eeb6de..2991c07ebc 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -62,11 +62,6 @@ struct ImportTaskRowView: View { Text("BIOS") .font(.subheadline) .foregroundColor(item.status.color) - } - else if let targetSystem = item.targetSystem() { - Text(targetSystem.name) - .font(.subheadline) - .foregroundColor(item.status.color) } else if !item.systems.isEmpty { Text("\(item.systems.count) systems") .font(.subheadline) From 90e5a6497d52a335c9ea26474c864a95ff8f3617 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 20:19:10 -0500 Subject: [PATCH 26/30] fixes pull to referesh --- .../Services/GameImporter/GameImporter.swift | 15 ++++++++---- .../PVGameLibraryUpdatesController.swift | 23 +++++++++---------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 98f31e6220..c77eaee799 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -339,9 +339,6 @@ public final class GameImporter: GameImporting, ObservableObject { defer { importQueueLock.unlock() } self.addImportItemToQueue(item) - - -// startProcessing() } public func addImports(forPaths paths: [URL]) { @@ -351,7 +348,17 @@ public final class GameImporter: GameImporting, ObservableObject { for path in paths { self.addImportItemToQueue(ImportQueueItem(url: path, fileType: .unknown)) } -// startProcessing() + } + + public func addImports(forPaths paths: [URL], targetSystem: PVSystem) { + importQueueLock.lock() + defer { importQueueLock.unlock() } + + for path in paths { + var item = ImportQueueItem(url: path, fileType: .unknown) + item.userChosenSystem = targetSystem + self.addImportItemToQueue(item) + } } // Public method to manually start processing if needed diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift index 88bcc73527..4260f7463e 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift @@ -274,12 +274,14 @@ public final class PVGameLibraryUpdatesController: ObservableObject { } } + /// auto scans ROM directories and adds to the import queue public func importROMDirectories() async { ILOG("PVGameLibrary: Starting Import") RomDatabase.reloadCache(force: true) RomDatabase.reloadFileSystemROMCache() let dbGames: [AnyHashable: PVGame] = await RomDatabase.gamesCache let dbSystems: [AnyHashable: PVSystem] = RomDatabase.systemCache + var queueGames = false for system in dbSystems.values { ILOG("PVGameLibrary: Importing \(system.identifier)") @@ -288,20 +290,17 @@ public final class PVGameLibraryUpdatesController: ObservableObject { dbGames.index(forKey: (system.identifier as NSString).appendingPathComponent($0.lastPathComponent)) == nil } if !newGames.isEmpty { - ILOG("PVGameLibraryUpdatesController: Importing \(newGames)") - //TODO: I think we want to add items to the import queue here - //await gameImporter.getRomInfoForFiles(atPaths: newGames, userChosenSystem: system.asDomain()) - #if os(iOS) || os(macOS) || targetEnvironment(macCatalyst) - await MainActor.run { - Task { - await self.addImportedGames(to: CSSearchableIndex.default(), database: RomDatabase.sharedInstance) - } - } - #endif + ILOG("PVGameLibraryUpdatesController: Adding \(newGames) to the queue") + gameImporter.addImports(forPaths: newGames, targetSystem:system) + queueGames = true } - ILOG("PVGameLibrary: Imported OK \(system.identifier)") + ILOG("PVGameLibrary: Added items for \(system.identifier) to queue") + } + if (queueGames) { + ILOG("PVGameLibrary: Queued new items, starting to process") + gameImporter.startProcessing() } - ILOG("PVGameLibrary: Import Complete") + ILOG("PVGameLibrary: importROMDirectories complete") } #if os(iOS) || os(macOS) || targetEnvironment(macCatalyst) From 5eaafc44fdccae3a315e86193050382cbe83d5c9 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 20:35:01 -0500 Subject: [PATCH 27/30] Fix game more info flow and (untested) db migration --- .../Importer/Services/GameImporter/GameImporter.swift | 4 ++++ .../Services/GameImporter/GameImporterDatabaseService.swift | 3 ++- .../PVGameLibraryViewController.swift | 5 +++-- .../PVUIBase/Game Library/PVGameMoreInfoViewController.swift | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index c77eaee799..4f666286b1 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -194,6 +194,10 @@ public final class GameImporter: GameImporting, ObservableObject { /// Path to the BIOS directory public var biosPath: URL { get { Paths.biosesPath }} + public var databaseService: GameImporterDatabaseServicing { + return gameImporterDatabaseService + } + /// Path to the conflicts directory public let conflictPath: URL = URL.documentsPath.appendingPathComponent("Conflicts/", isDirectory: true) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 633717c0ad..de65e7262f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -21,11 +21,12 @@ import PVRealm import Perception import SwiftUI -protocol GameImporterDatabaseServicing { +public protocol GameImporterDatabaseServicing { func setOpenVGDB(_ vgdb: OpenVGDB) func setRomsPath(url:URL) func importGameIntoDatabase(queueItem: ImportQueueItem) async throws func importBIOSIntoDatabase(queueItem: ImportQueueItem) async throws + func getUpdatedGameInfo(for game: PVGame, forceRefresh: Bool) -> PVGame func getArtwork(forGame game: PVGame) async -> PVGame } diff --git a/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift b/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift index ccf4b3d364..c56a24cc71 100644 --- a/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/CollectionViewController/PVGameLibraryViewController.swift @@ -1010,8 +1010,9 @@ public final class PVGameLibraryViewController: GCEventViewController, UITextFie self.hud.hide(animated:true, afterDelay: 1.0) Task.detached(priority: .utility) { do { - //TODO: fix this - //try await self.gameImporter.importFiles(atPaths: paths) + //unclear if this would actually work... + self.gameImporter.addImports(forPaths: paths) + self.gameImporter.startProcessing() } catch { ELOG("Error: \(error.localizedDescription)") } diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift index 6a9996cacb..b8a57279fc 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameMoreInfoViewController.swift @@ -707,7 +707,7 @@ public final class PVGameMoreInfoViewController: PVGameMoreInfoViewControllerBas Task { [weak self] in guard let self = self else { return } //TODO: fix this - //self.game = GameImporter.shared.lookupInfo(for: game, overwrite: false) + self.game = GameImporter.shared.databaseService.getUpdatedGameInfo(for: game, forceRefresh: false) } } } From d5d423d09a9bb6345ae6b7728db01b66cb72d57b Mon Sep 17 00:00:00 2001 From: David Proskin Date: Sun, 10 Nov 2024 21:20:18 -0500 Subject: [PATCH 28/30] swipe to delete works now --- .../Services/GameImporter/GameImporter.swift | 19 +++++++++++++++++++ .../GameImporterFileService.swift | 10 ++++++++++ .../PVSwiftUI/Imports/ImportStatusView.swift | 19 ++++++++++--------- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 4f666286b1..038dacf01c 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -113,6 +113,7 @@ public protocol GameImporting { func addImport(_ item: ImportQueueItem) func addImports(forPaths paths: [URL]) + func removeImports(at offsets: IndexSet) func startProcessing() func importQueueContainsDuplicate(_ queue: [ImportQueueItem], ofItem queueItem: ImportQueueItem) -> Bool @@ -364,6 +365,24 @@ public final class GameImporter: GameImporting, ObservableObject { self.addImportItemToQueue(item) } } + + public func removeImports(at offsets: IndexSet) { + importQueueLock.lock() + defer { importQueueLock.unlock() } + + for index in offsets { + let item = importQueue[index] + + // Try to delete the associated file + do { + try gameImporterFileService.removeImportItemFile(item) + } catch { + ELOG("removeImports - Failed to delete file at \(item.url): \(error)") + } + } + + importQueue.remove(atOffsets: offsets) + } // Public method to manually start processing if needed public func startProcessing() { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift index 99d9fcf846..5730fd701f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterFileService.swift @@ -12,6 +12,7 @@ import RealmSwift protocol GameImporterFileServicing { func moveImportItem(toAppropriateSubfolder queueItem: ImportQueueItem) async throws func moveToConflictsFolder(_ queueItem: ImportQueueItem, conflictsPath: URL) async throws + func removeImportItemFile(_ importItem: ImportQueueItem) throws } class GameImporterFileService : GameImporterFileServicing { @@ -138,6 +139,15 @@ class GameImporterFileService : GameImporterFileServicing { return destination } + func removeImportItemFile(_ importItem: ImportQueueItem) throws { + let fileManager = FileManager.default + + // If file exists at destination, remove it first + if fileManager.fileExists(atPath: importItem.url.path) { + try fileManager.removeItem(at: importItem.url) + } + } + /// Moves a file and overwrites if it already exists at the destination public func moveAndOverWrite(sourcePath: URL, destinationPath: URL) throws -> URL { let fileManager = FileManager.default diff --git a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift index 2991c07ebc..506373418c 100644 --- a/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift +++ b/PVUI/Sources/PVSwiftUI/Imports/ImportStatusView.swift @@ -90,8 +90,6 @@ struct ImportTaskRowView: View { } .padding() .background(Color.white) - .cornerRadius(10) - .shadow(color: .gray.opacity(0.2), radius: 5, x: 0, y: 2) .onTapGesture { if item.status == .conflict { isNavigatingToSystemSelection = true @@ -111,20 +109,23 @@ struct ImportStatusView: View { var gameImporter:GameImporter weak var delegate:ImportStatusDelegate! + private func deleteItems(at offsets: IndexSet) { + gameImporter.removeImports(at: offsets) + } + var body: some View { NavigationView { - ScrollView { + List { if gameImporter.importQueue.isEmpty { Text("No items in the import queue") .foregroundColor(.gray) .padding() } else { - LazyVStack(spacing: 10) { - ForEach(gameImporter.importQueue) { item in - ImportTaskRowView(item: item).id(item.id) - } - } - .padding() + ForEach(gameImporter.importQueue) { item in + ImportTaskRowView(item: item).id(item.id) + }.onDelete( + perform: deleteItems + ) } } .navigationTitle("Import Status") From 2ea4cf1e93041582f510cc0ded67cf1d1d99d897 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Mon, 11 Nov 2024 11:32:03 -0500 Subject: [PATCH 29/30] fixes a realm threading crash when importing roms into the DB --- .../GameImporter/GameImporterDatabaseService.swift | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index de65e7262f..46d9263b8f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -195,7 +195,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if game.originalArtworkFile == nil { game = await getArtwork(forGame: game) } - self.saveGame(game) + await self.saveGame(game) } @discardableResult @@ -485,10 +485,14 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } /// Saves a game to the database - func saveGame(_ game:PVGame) { + func saveGame(_ game:PVGame) async { do { - //TODO: this might crash if not on main thread - validate let database = RomDatabase.sharedInstance + let realm = try! await RomDatabase.sharedInstance.realm + + if let system = realm.object(ofType: PVSystem.self, forPrimaryKey: game.systemIdentifier) { + game.system = system + } try database.writeTransaction { database.realm.create(PVGame.self, value:game, update:.modified) } From b213e7472a4662a4d6147f2752a6f909d3253c02 Mon Sep 17 00:00:00 2001 From: David Proskin Date: Mon, 11 Nov 2024 11:39:31 -0500 Subject: [PATCH 30/30] adds in auto-start for the queue after the processing of files for addition to the queue. seems to work? --- .../PVUIBase/Game Library/PVGameLibraryUpdatesController.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift index 4260f7463e..8cadfdb8b0 100644 --- a/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift +++ b/PVUI/Sources/PVUIBase/Game Library/PVGameLibraryUpdatesController.swift @@ -389,6 +389,9 @@ public final class PVGameLibraryUpdatesController: ObservableObject { gameImporter.addImports(forPaths: otherFiles) DLOG("Finished importing other files") } + + //it seems reasonable to kick off the queue here + gameImporter.startProcessing() } private func setupBIOSObserver() {