changes towards the next release: (#62)

* Update 'formats' in peas.json from RP platforms.cfg file * tests fixed * Update gitignore * added example switches for PDF scraping * Avoid irritating errormessage on a pristine checkout build * config.ini onlyMissing added and counting of Roms logic for that flag * refactorings and cleanups * Extra hint when unsupported platform is provided * update changelog
Gemba · May 12, 2024 · e337cb2 · e337cb2
1 parent f5a6bd9
commit e337cb2
Show file tree

Hide file tree

Showing 23 changed files with 377 additions and 415 deletions.
diff --git a/.gitignore b/.gitignore
@@ -69,3 +69,4 @@ build
 
 # VS Code
 *.code-workspace
+.vscode
diff --git a/README.md b/README.md
@@ -118,7 +118,7 @@ If you want to build the latest `main/HEAD` version use the following commands.
 ```
 git clone --depth 1 https://github.com/Gemba/skyscraper.git
 cd skyscraper
-make --ignore-errors clean
+[[ -f Makefile ]] && make --ignore-errors clean
 rm --force .qmake.stash
 QT_SELECT=5 qmake 
 make -j$(nproc)

diff --git a/config.ini.example b/config.ini.example
@@ -74,6 +74,7 @@
 ;spaceCheck="true"
 ;scummIni="/full/path/to/scummvm.ini"
 ;tidyDesc="true"
+;onlyMissing="false"
 
 ; The following is an example of configs that only affect the 'snes' platform.
 ;[snes]

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -8,6 +8,15 @@
   [`manuals=true`](CONFIGINI.md#manuals) and
   [`gameListVariants=enable-manuals`](CONFIGINI.md#gamelistvariants). Thanks for
   the initial PR, @pandino
+- Added: Present also release year as hint. This is useful when in interactive
+  mode and more than one game with the same name is found. Skyscraper can be
+  guided to prefer a specific game when the release year is added in parenthesis
+  as part of the ROM name (or alias in `aliasMap.csv`). Thanks @mjkaye
+- Changed: Persistent config option for counting and scraping only games which
+  have no game data in the cache. This is a commodity to the already existing
+  flag with the same name. If you use a scraper with a scraping limit for games
+  to be scraped at once (e.g. MobyGames). Documentation
+  [here](CONFIGINI.md#onlymissing). Thanks for the suggestion, @sleve_mcdichael
 
 ### Version 3.11.0 (2024-04-15)
 

diff --git a/docs/CONFIGINI.md b/docs/CONFIGINI.md
@@ -101,6 +101,7 @@ This is an alphabetical index of all configuration options including the section
 | [mediaFolderHidden](CONFIGINI.md#mediafolderhidden)         |          |                |       Y        |               |
 | [minMatch](CONFIGINI.md#minmatch)                           |    Y     |       Y        |                |       Y       |
 | [nameTemplate](CONFIGINI.md#nametemplate)                   |    Y     |       Y        |                |               |
+| [onlyMissing](CONFIGINI.md#onlymissing)                     |    Y     |       Y        |                |       Y       |
 | [platform](CONFIGINI.md#platform)                           |    Y     |                |                |               |
 | [pretend](CONFIGINI.md#pretend)                             |    Y     |       Y        |                |               |
 | [region](CONFIGINI.md#region)                               |    Y     |       Y        |                |               |
@@ -793,6 +794,19 @@ Allowed in sections: `[main]`, `[<PLATFORM>]`
 
 ---
 
+#### onlyMissing
+
+This flag, when set true, will only scrape game information for ROMs that do not have any data in Skyscraper's cache. The cache information of a ROM that has a cache hit will not be updated or extended when 'only missing' is true. A use case could be that you want to exclude already scraped games from a scraping process and you are using a scraping module which has a limit on how many ROMs may be scraped: With 'only missing' enabled the ROMs with cache data are subtracted from all ROMs to be scraped for a platform, thus the remaining ROM count may be below the scrape module's limit.
+
+!!! tip
+
+    Please consider using the command line flag option `--flags onlymissing` instead. It makes little sense to set this permanently in your configuration unless you _always_ want to use the cached data for already cached game entries. Also when 'only missing' is set and the cache has any data for a game entry, the refresh option has no effect. But you still can use the 'only missing' in the config file if you absolutely want to.
+
+Default value: false  
+Allowed in sections: `[main]`, `[<PLATFORM>]`, `[<SCRAPER>]`
+
+---
+
 #### startAt
 
 If you only ever wish to gather data for a subset of your roms from the scraping modules you can use this option to set the starting rom. It will then scrape alphabetically from that rom and onwards.

diff --git a/peas.json b/peas.json
@@ -63,12 +63,15 @@
         "formats": [
             "*.adf",
             "*.adz",
+            "*.chd",
             "*.cue",
             "*.dms",
             "*.hdf",
             "*.img",
+            "*.ipf",
             "*.iso",
             "*.lha",
+            "*.m3u",
             "*.rp9",
             "*.uae"
         ],
@@ -83,8 +86,12 @@
             "cpc"
         ],
         "formats": [
+            "*.cdt",
             "*.cpc",
-            "*.dsk"
+            "*.cpr",
+            "*.dsk",
+            "*.m3u",
+            "*.tap"
         ],
         "scrapers": [
             "screenscraper"
@@ -95,7 +102,9 @@
             "apple ii"
         ],
         "formats": [
-            "*.dsk"
+            "*.dsk",
+            "*.nib",
+            "*.po"
         ],
         "scrapers": [
             "screenscraper"
@@ -183,7 +192,10 @@
         ],
         "formats": [
             "*.bin",
-            "*.dat"
+            "*.cue",
+            "*.dat",
+            "*.fba",
+            "*.iso"
         ],
         "scrapers": [
             "arcadedb",
@@ -195,7 +207,9 @@
             "arcadia 2001",
             "emerson arcadia 2001"
         ],
-        "formats": [],
+        "formats": [
+            "*.bin"
+        ],
         "scrapers": [
             "screenscraper"
         ]
@@ -280,10 +294,14 @@
             "*.a52",
             "*.atr",
             "*.atr.gz",
+            "*.atx",
             "*.bas",
             "*.bin",
             "*.car",
+            "*.cas",
+            "*.com",
             "*.dcm",
+            "*.rom",
             "*.xex",
             "*.xfd",
             "*.xfd.gz"
@@ -343,6 +361,7 @@
             "*.ctr",
             "*.img",
             "*.ipf",
+            "*.m3u",
             "*.raw",
             "*.rom",
             "*.st",
@@ -444,9 +463,14 @@
             "commodore c64/128"
         ],
         "formats": [
+            "*.cmd",
             "*.crt",
             "*.d64",
+            "*.d71",
+            "*.d80",
+            "*.d81",
             "*.g64",
+            "*.m3u",
             "*.prg",
             "*.t64",
             "*.tap",
@@ -572,7 +596,9 @@
         "aliases": [
             "CreatiVision"
         ],
-        "formats": [],
+        "formats": [
+            "*.rom"
+        ],
         "scrapers": [
             "mobygames",
             "screenscraper",
@@ -586,7 +612,8 @@
             "pioneer laseractive"
         ],
         "formats": [
-            "*.daphne"
+            "*.daphne",
+            "*.ogv"
         ],
         "scrapers": [
             "screenscraper"
@@ -709,7 +736,11 @@
             "video system co.",
             "visco"
         ],
-        "formats": [],
+        "formats": [
+            "*.cue",
+            "*.fba",
+            "*.iso"
+        ],
         "scrapers": [
             "arcadedb",
             "screenscraper"
@@ -777,7 +808,9 @@
             "sega game gear"
         ],
         "formats": [
-            "*.gg"
+            "*.bin",
+            "*.gg",
+            "*.sms"
         ],
         "scrapers": [
             "screenscraper"
@@ -826,6 +859,7 @@
             "nintendo gamecube"
         ],
         "formats": [
+            "*.ciso",
             "*.cso",
             "*.gcm",
             "*.gcz",
@@ -855,7 +889,9 @@
         "aliases": [],
         "formats": [
             "*.bin",
-            "*.int"
+            "*.int",
+            "*.itv",
+            "*.rom"
         ],
         "scrapers": [
             "screenscraper"
@@ -886,7 +922,8 @@
         "formats": [
             "*.dsk",
             "*.img",
-            "*.rom"
+            "*.rom",
+            "*.txt"
         ],
         "scrapers": [
             "mobygames",
@@ -1204,6 +1241,7 @@
             "sega master system"
         ],
         "formats": [
+            "*.bin",
             "*.sms"
         ],
         "scrapers": [
@@ -1270,8 +1308,10 @@
             "msx2+"
         ],
         "formats": [
+            "*.cas",
             "*.col",
             "*.dsk",
+            "*.m3u",
             "*.mx1",
             "*.mx2",
             "*.rom"
@@ -1438,7 +1478,12 @@
             "neo-geo",
             "neo-geo cd"
         ],
-        "formats": [],
+        "formats": [
+            "*.chd",
+            "*.cue",
+            "*.fba",
+            "*.iso"
+        ],
         "scrapers": [
             "arcadedb",
             "openretro",
@@ -1535,8 +1580,8 @@
         ],
         "formats": [
             "*.img",
-            "*.prc",
-            "*.pqa"
+            "*.pqa",
+            "*.prc"
         ],
         "scrapers": [
             "mobygames",
@@ -1558,8 +1603,17 @@
             "*.bat",
             "*.com",
             "*.conf",
+            "*.cue",
+            "*.dosz",
             "*.exe",
-            "*.sh"
+            "*.ima",
+            "*.img",
+            "*.ins",
+            "*.iso",
+            "*.m3u",
+            "*.m3u8",
+            "*.sh",
+            "*.vhd"
         ],
         "scrapers": [
             "thegamesdb"
@@ -1574,6 +1628,7 @@
             "*.88d",
             "*.cmt",
             "*.d88",
+            "*.m3u",
             "*.t88"
         ],
         "scrapers": [
@@ -1620,6 +1675,7 @@
             "turbografx-16/pc engine cd"
         ],
         "formats": [
+            "*.ccd",
             "*.chd",
             "*.cue",
             "*.pce"
@@ -1660,7 +1716,8 @@
             "*.chd",
             "*.cue",
             "*.img",
-            "*.iso"
+            "*.iso",
+            "*.pce"
         ],
         "scrapers": [
             "screenscraper"
@@ -1739,6 +1796,7 @@
             "sony playstation 2"
         ],
         "formats": [
+            "*.bin",
             "*.bz2",
             "*.chd",
             "*.cso",
@@ -1989,6 +2047,7 @@
         ],
         "formats": [
             "*.bin",
+            "*.bs",
             "*.fig",
             "*.mgd",
             "*.sfc",
@@ -2185,9 +2244,12 @@
             "wiiware"
         ],
         "formats": [
+            "*.ciso",
             "*.cso",
+            "*.gcm",
             "*.gcz",
             "*.iso",
+            "*.m3u",
             "*.rvz",
             "*.wad",
             "*.wbfs"
@@ -2262,8 +2324,13 @@
             "sharp x68000"
         ],
         "formats": [
+            "*.2hd",
+            "*.d88",
             "*.dim",
-            "*.m3u"
+            "*.hdf",
+            "*.hdm",
+            "*.m3u",
+            "*.xdf"
         ],
         "scrapers": [
             "screenscraper"
@@ -2333,6 +2400,7 @@
             "*.gz",
             "*.img",
             "*.mgt",
+            "*.rzx",
             "*.scl",
             "*.sna",
             "*.szx",
@@ -2347,4 +2415,4 @@
             "worldofspectrum"
         ]
     }
-}
+}