From ebfb03c0a6cc8c7ba2614bcaae45c9c3bb330f57 Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Fri, 13 Nov 2026 20:16:26 +0530 Subject: [PATCH] BulkImageScrape: add configurable delay between image scrapes The plugin always slept a hardcoded 0.5 seconds before scraping each image, which is not enough for scrapers hitting rate-limited websites and cannot be changed without editing the plugin on disk. Add a ScrapeDelaySeconds NUMBER setting. When unset the previous 0.5 second delay is kept, 0 disables the delay, negative values are clamped to 0 and non-numeric/non-finite values fall back to the default with a warning. Document the setting in the README and bump the plugin version. Closes #736 --- plugins/bulkImageScrape/BulkImageScrape.yml | 7 +++-- plugins/bulkImageScrape/README.md | 4 +++ plugins/bulkImageScrape/bulkImageScrape.py | 35 +++++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/plugins/bulkImageScrape/BulkImageScrape.yml b/plugins/bulkImageScrape/BulkImageScrape.yml index 76c2756a..6805ba51 100755 --- a/plugins/bulkImageScrape/BulkImageScrape.yml +++ b/plugins/bulkImageScrape/BulkImageScrape.yml @@ -1,7 +1,7 @@ name: Bulk Image Scrape # requires: PythonDepManager description: Apply an image scraper to all images -version: 0.3.6 +version: 0.3.7 url: https://discourse.stashapp.cc/t/bulk-image-scrape/1339 exec: - python @@ -36,7 +36,10 @@ settings: PerformerIdsFilter: displayName: List of performers IDs (comma separated) to filter images the plugins sees type: STRING - + ScrapeDelaySeconds: + displayName: Delay in seconds between each image scrape (default 0.5, use a higher value to avoid rate limits) + type: NUMBER + tasks: - name: "Bulk Image Scrape" description: Apply an image scraper to all images diff --git a/plugins/bulkImageScrape/README.md b/plugins/bulkImageScrape/README.md index d52ce351..b4100531 100644 --- a/plugins/bulkImageScrape/README.md +++ b/plugins/bulkImageScrape/README.md @@ -44,6 +44,10 @@ It is mandatory to enter the Scraper ID of the Scraper you want to use. In this > Any image that has one or more of the Tags from this setting will be skipped by the plugin if Tags are specified here. Multiple Tags must be comma separated. If the plugin can't find a tag you specified it will notify you in the logs +- `Delay in seconds between each image scrape` + +> Time the plugin waits before scraping each image. Defaults to `0.5` seconds when not set. Increase it (for example to `5`) if the scraper's website rate limits you, or set it to `0` to disable the delay. Fractions such as `1.5` are allowed. Negative values are treated as `0` and invalid values fall back to the default + ## Task After you adapted the config to your liking and made sure your image scraper of choice works properly simply start the task in `Settings > Tasks` diff --git a/plugins/bulkImageScrape/bulkImageScrape.py b/plugins/bulkImageScrape/bulkImageScrape.py index b72bf37e..6120167a 100755 --- a/plugins/bulkImageScrape/bulkImageScrape.py +++ b/plugins/bulkImageScrape/bulkImageScrape.py @@ -125,6 +125,35 @@ def parse_performerIds_filter(performerIdsFilter: str) -> list[str]: return performerIdsFilter +DEFAULT_SCRAPE_DELAY_SECONDS: float = 0.5 + + +def parse_scrape_delay(value: object) -> float: + """ + Parse the delay (in seconds) to wait before each image is scraped. + + Falls back to the default for missing or non-numeric values and never + returns a negative delay. + """ + if value is None or value == "" or isinstance(value, bool): + return DEFAULT_SCRAPE_DELAY_SECONDS + + try: + delay = float(value) + except (TypeError, ValueError): + delay = math.nan + + # NaN and infinity would break time.sleep() + if not math.isfinite(delay): + log.warning( + f"Invalid ScrapeDelaySeconds value {value!r} - " + f"using the default of {DEFAULT_SCRAPE_DELAY_SECONDS} seconds" + ) + return DEFAULT_SCRAPE_DELAY_SECONDS + + return max(delay, 0.0) + + def build_image_filter(skip_tags: list[str], performerIdsFilter: list[str], exclude_organized: bool ) -> dict: image_filter: dict = {} @@ -306,7 +335,7 @@ def update_image(client: StashInterface, update: dict) -> dict | None: "MergeExistingTags": False, "ExcludeOrganized": False, "SkipEntriesNum": 0, - + "ScrapeDelaySeconds": DEFAULT_SCRAPE_DELAY_SECONDS, } if "BulkImageScrape" in config: @@ -329,6 +358,7 @@ def update_image(client: StashInterface, update: dict) -> dict | None: scraper_id: str = validate_scraper(stash, settings["ScraperID"]) parsed_skip_tags: list[str] = parse_skip_tags(stash, settings["SkipTags"]) parsed_performerids_filter: list[str] = parse_performerIds_filter(settings["PerformerIdsFilter"]) +scrape_delay: float = parse_scrape_delay(settings["ScrapeDelaySeconds"]) # # MAIN @@ -350,7 +380,8 @@ def update_image(client: StashInterface, update: dict) -> dict | None: ) for i, image in enumerate(images, start=1): - time.sleep(0.5) + if scrape_delay > 0: + time.sleep(scrape_delay) log.progress((i / total_images)) log.debug(f"Scraping image {image['id']} with scraper {scraper_id}")